\batchmode
\documentstyle[12pt]{book}
\makeatletter


 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
%













\newcounter{bibliosection}

  


\newcounter{example}


\newcounter{exercise}





  





  






















































































\input psfig   %








\def\CE#1#2{{{E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpi#1#2{{{E_\pi}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpip#1#2{{{E_{\pi^\prime}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpistar#1#2{{{E_{\pi^*}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEggpistar#1#2{{{E_{\pi^*}}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEggpi#1#2{{{E_\pi}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEgpi#1#2{{{E_\pi}\!\Bigl\{#1\Bigm|#2\Bigr\}}}

\def\hatCE#1#2{{\hat {E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEgg#1#2{{{E}\biggl\{#1\biggm|#2\biggr\}}}

\def\E#1{{{E}\left\{#1\right\}}}

\def\Epi#1{{{E_\pi}\!\left\{#1\right\}}}

\def\Eggpi#1{{{E_\pi}\!\biggl\{#1\biggr\}}}

\def\Epip#1{{{E_{\pi^\prime}}\!\left\{#1\right\}}}

\def\Ea#1{{{E_a}\!\left\{#1\right\}}}

\def\Egg#1{{{E}\biggl\{#1\biggr\}}}

\def\CP#1#2{{{Pr}\left\{{#1}{\mid}{#2}\right\}}}

\def\Pr#1{{{Pr}\left\{#1\right\}}}

\def\w{{\bf w}}

\def\cross{\times}

\def\grad{\nabla}

\def\first#1#2{{\partial #1\over \partial #2}}

\def\second#1#2{{\partial^2 #1\over \partial #2^2}}

\def\implies{\Rightarrow}

\def\union{\cup}

\def\bibliosection#1{%
  \addtocounter{bibliosection}{1}%
  \medskip\goodbreak
  {\bf \arabic{chapter}.\arabic{bibliosection} #1}
  \smallskip}

\def\skipbibliosection{\addtocounter{bibliosection}{1}}

\def\example{
  \addtocounter{example}{1}
  \smallskip
  \noindent
  {\bf Example \arabic{chapter}.\arabic{example}}~~}

\def\exercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}}~~ }

\def\pexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}~(programming)~~}}

\def\starredexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}*~}}

\def\lastexample{\arabic{chapter}.\arabic{example}}

\def\lastexercise{\arabic{chapter}.\arabic{exercise}}

\def\resetmycounters{
  \setcounter{bibliosection}{0}
  \setcounter{example}{0}
  \setcounter{exercise}{0}}

\def\myfootnote#1{\footnote{#1}}

\def\lambdatarget{7.3}

\def\polefigure{3.2}

\def\StateAggregation{8.4}

\def\banditchap{2}

\def\probchap{3}

\def\DPchap{4}

\def\MCchap{5}

\def\TDchap{6}

\def\ETchap{7}

\def\FAchap{8}

\def\modelchap{9}

\def\CSchap{11}

\def\Sumchap{10}

\def\finalchap{10}

\def\eqincN{\banditchap.4}

\def\eqAverage{\banditchap.1}

\def\eqSAconditions{\banditchap.7}

\def\secinc{\banditchap.5}

\def\secnonstationary{\banditchap.6}

\def\softmaxsection{\banditchap.3}

\def\RCsection{2.8}

\def\predictionbackups{\probchap.4}

\def\DPbackupVpi{\probchap.4a}

\def\undiscountedsection{\probchap.?}

\def\maxbackups{\probchap.7}

\def\BellmanVpi{\probchap.10}

\def\PImprov{\DPchap.2}

\def\GPIsec{\DPchap.6}

\def\EstimateOtherPolicySec{\MCchap.5}

\def\TDprediction{\TDchap.2}

\def\simpleTDrule{\TDchap.2}

\def\randomwalkexample{\TDchap.2}

\def\randomwalkfigure{\TDchap.4}

\def\sarsasection{\TDchap.3}

\def\ACsection{\TDchap.6}

\def\Rlearningsection{\TDchap.7}

\def\afterstatesection{\TDchap.8}

\def\ACrule{\TDchap.8}

\def\ACruleplus{\TDchap.9}

\def\TileCodingSection{\FAchap.3.2}

\def\sarsalalg{\FAchap.8}

\def\lambdacomparison{\FAchap.10}

\def\heuristicsearchsection{\modelchap.7}

\def\planspacesection{\modelchap.10}

\def\uniformvsonpolicysection{\modelchap.9}

\def\ZDcasestudy{\CSchap.6}

\def\samuelsection{\CSchap.2}

\def\TDGammonSection{\CSchap.1}

\def\g{\mbox{$\gamma$}}

\def\a{\mbox{$\alpha$}}

\def\l{\mbox{$\lambda$}}

\def\e{\mbox{$\epsilon$}}

\def\ee{\mbox{$\varepsilon$}}

\def\n{\mbox{$n$}}

\def\r{\mbox{$R$}}

\def\rl{R^{\lambda}}

\def\sp{{s^\prime}}

\def\ap{{a^\prime}}

\def\TDl{TD($\lambda$)}

\def\itTDl{{\it TD\/}\mbox{\rm ($\lambda$)}}

\def\Ql{Q($\lambda$)}

\def\itQl{{\it Q\/}\mbox{\rm ($\lambda$)}}

\def\sarsal{Sarsa($\lambda$)}

\def\itsarsal{{\it Sarsa\/}\mbox{\rm ($\lambda$)}}

\def\={\!=\!}

\def\X{{\cal X}}

\def\A{{\cal A}}

\def\S{{\cal S}}

\def\P{{\cal P}}

\def\R{{\cal R}}

\def\Pssa{\P_{s\sp}^a}

\def\Rssa{\R_{s\sp}^a}

\def\endofexample{$\diamondsuit$}

\def\starredsection#1{\section{#1\ (*)}}

\def\&{and}

\def\figbody#1#2#3#4#5{%








\centerline{\psfig{figure=#1.ps,rheight=#3,#5}}}

\def\comment#1{}

\def\boxit#1{\vbox{\hrule\hbox{\vrule\kern3pt
             \vbox{\kern3pt#1\kern3pt}\kern3pt\vrule}\hrule}}

\def\myboxit#1{\vbox{\hrule\hbox{\vrule\kern6pt
             \vbox{\kern-17pt#1\kern-12pt}\kern8pt\vrule}\hrule}}

\def\DVl{\Delta V^{\lambda}}

\def\DVTD{\Delta V^{TD}}

\def\I{{\cal I}}

\def\non{\nonumber}

\def\hsp1{\hspace*{.55in}}

\makeatother
\newenvironment{tex2html_wrap}{}{}
\newwrite\lthtmlwrite
\def\lthtmltypeout#1{{\let\protect\string\immediate\write\lthtmlwrite{#1}}}%
\newbox\sizebox
\begin{document}
\pagestyle{empty}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1718: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1722
$\alpha $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1722: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1726: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\unhbox \voidb@x \hbox {$\lambda $}=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1728: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\unhbox \voidb@x \hbox {$\lambda $}=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1730: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1732: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1736: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1738: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1740: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1742: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1744: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1746: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1748: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1750: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1752: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1754: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1756: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1758: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1760
$\alpha $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1760: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1762: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1764: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\setcounter{bibliosection}{0}
\setcounter{example}{0}
\setcounter{exercise}{0}
\setcounter{chapter}{6}
\stepcounter{chapter}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1766: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1768: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1770: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1782: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1784: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1788: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=n-step-backups.ps,rheight=2.6in,width=3.852in}}


\label{fig:n-step-backups}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1804: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t,r_{t+1},s_{t+1},r_{t+2},\ldots,r_T,s_T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1806: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1808: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1810: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\r_t = r_{t+1} + \mbox{$\gamma$}
  r_{t+2} + \g^2 r_{t+3} + \cdots + \g^{T-t-1} r_T, \end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{displaymath}\r^{(1)}_t = r_{t+1} + \mbox{$\gamma$}
  V_t(s_{t+1}). \end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
  V_t(s_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1818: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
  r_{t+2} + \g^2 r_{t+3} + \cdots + \g^{T-t-1} r_T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1820: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\r^{(2)}_t = r_{t+1} + \mbox{$\gamma$}
  r_{t+2} + \g^2 V_{t}(s_{t+2}), \end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\g^2 V_{t}(s_{t+2})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1824: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\g^2 r_{t+3} + \g^3
r_{t+4} + \cdots + \g^{T-t-1} r_T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1826: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\r^{(n)}_t = r_{t+1} + \mbox{$\gamma$}
  r_{t+2} + \g^2 + \cdots + \g^{n-1} r_{t+n} +
            \g^{n} V_{t}(s_{t+n}).
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r^{(n)}_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1838: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r_t^{(n)}=\r_t^{(T-t)}=\r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1850: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1862: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1864: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1870: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 1775
\Delta V_t(s_t) = \mbox{$\alpha$}
  \left[\r^{(n)}_t - V_t(s_t)\right]. \end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta V_t(s)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1874: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\not=s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1876: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_{t+1}(s)=V_t(s)+\Delta V_t(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1880: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1882: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1884: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1888: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)+\sum_{t=0}^{T-1} \Delta
V_t(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1890: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1900: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\g^n$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1904: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:nstep-error-correction}
 \max_s \left|{{E_\pi}\!\Bigl\{{\r^{(n)}_t}\Bigm|s_t\=s\Bigr\}}
-V^\pi(s)\right| ~\le~
 \g^n \, \max_s \left|V(s)-V^\pi(s)\right|.
\end{equation}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1994
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1995
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_0(s)=0.5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1936: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(E)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1938: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(D)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1946: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(E)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1948: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1964
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1964: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1970: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1974
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1974: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1974
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1974: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=nstep-results.ps,rheight=5in,width=4.212in}}


\label{fig:nstep-results}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1996
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1997
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1998
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1999
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2000
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2001
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1982
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1982: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2004: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r^{ave}_t={1\over 2} \r^{(2)}_t + {1\over 2} \r^{(4)}_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2010: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=mixed-backup.ps,rheight=2in,width=.64in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2014: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2016: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2018: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2014: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2016: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2018: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}[tbp]

\centerline{\psfig{figure=TDlbackup.ps,rheight=2.7in,width=2.418in}}

\vspace*{-.1in}

\label{fig:TDlbackup}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2020: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\l^{n-1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2026: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$0\le\mbox{$\lambda$}
 \le
1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2028: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\l$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2030: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2032: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\rl_t = (1-\mbox{$\lambda$}
 ) \sum_{n=1}^\infty \l^{n-1} \r^{(n)}_t. \end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\l$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2036: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(1-\mbox{$\lambda$}
 )\l$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2038: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(1-\mbox{$\lambda$}
 )\l^2$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2040: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2042: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2046: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:lambdareturn}
\rl_t ~~=~~ (1-\mbox{$\lambda$}
 )\! \sum_{n=1}^{T-t-1} \l^{n-1} \r^{(n)}_t ~~+~~ \l^{T-t-1}\r_t.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2050: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2052: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2054: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2056: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2058
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2058: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2060: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2062: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r^{(1)}_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2064: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2066: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2068: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2070: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2070: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=lambdaweighting.ps,rheight=1.9in,width=5.145in}}


\label{fig:lambdaweighting}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2074: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2076: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta V_t(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2080: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 398
\label{eq:lreturnalg}
 \Delta V_t(s_t) = \mbox{$\alpha$}
  \left[ \rl_t - V_t(s_t) \right].
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta V_t(s)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2084: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\not=s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2086: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}[btp]

\centerline{\psfig{figure=forward-view.ps,rheight=1.3in,width=3.948in}}


\label{fig:forward-view}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2090: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2092: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2094: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2096: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2098: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2100: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2102: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2108: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2112: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2114: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2156
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2157
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2116: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2118: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2124: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2128
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2128: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2130: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2132: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2134: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2134: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=lreturn-results.ps,rheight=2.5in,width=4.041in}}

\vspace*{-.2in}

\label{fig:lreturn-results}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2158
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2159
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2136: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2138: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2140: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2142: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau_\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2144: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2160: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2162: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2164: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2166: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2168: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e_t(s)\in\Re^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2174: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2176: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:Atrace}
e_{t}(s) = \left\{\begin{array}{ll}
  \gamma\lambda\, e_{t-1}(s)      & \mbox{if $s\!\not=\!s_t$};\\ 
  \gamma\lambda\, e_{t-1}(s) + 1  & \mbox{if $s\=s_t$},\end{array} \right.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2186: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2188: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2190: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=Atrace.ps,rheight=0.904in,width=4.456in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2192: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:TDerror}
 \delta_t ~=~ r_{t+1} + \mbox{$\gamma$}
  V_t(s_{t+1}) - V_t(s_t).
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2196: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 454
\label{eq:TDl}
 \Delta V_t(s) = \mbox{$\alpha$}
 \, \delta_t\, e_t(s), \qquad \mbox{for all~} s\in\S.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2200: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2202: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2204: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2206: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2208: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \mbox{action given by $\pi$ for $s$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2212: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2218: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta \gets r + \mbox{$\gamma$}
  V({s^\prime}
 ) - V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2220: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s) \gets e(s) + 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2222: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2226
$V(s) \gets V(s) + \alpha\delta e(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2226: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s) \gets \mbox{$\gamma$}
 \mbox{$\lambda$}
 e(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2228: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2230: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2234: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2236: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2234: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2236: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:online-TDl}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2238: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}[btp]

\centerline{\psfig{figure=backward-view.ps,rheight=1.8in,width=3.948in}}


\label{fig:backward-view}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2240: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2242: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2246: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2248: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2250: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 <1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2252: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2254: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2256: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2258: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2260: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\g^k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2264: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2266: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2268: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2270: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2286: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2288: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2290: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\DVl_t(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2292: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2296: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2298: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\DVTD_t(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2300: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2306: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:goal}
 \sum_{t=0}^{T-1} \DVTD_t(s) = \sum_{t=0}^{T-1} \DVl_t(s_t) \I_{ss_t},
 \qquad \mbox{for all~} s\in\S,
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\I_{s{s^\prime}
 }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2308: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\!=\!
 \sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2312: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}%
e_t(s) = \sum_{k=0}^t (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^{t-k} \I_{ss_k}.
\end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{eqnarray}% latex2html id marker 509
\label{eq:leftside}
\sum_{t=0}^{T-1}\DVTD_t(s)
&=& \sum_{t=0}^{T-1} \alpha\delta_t\sum_{k=0}^t (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^{t-k}\I_{ss_k}\nonumber \\ 
&=& \sum_{k=0}^{T-1} \alpha\delta_k\sum_{t=0}^k (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^{k-t}\I_{ss_t}\nonumber \\ 
&=& \sum_{k=t}^{T-1} \alpha\delta_k\sum_{t=0}^{T-1} (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^{k-t}\I_{ss_t}\nonumber \\ 
&=& \sum_{t=0}^{T-1} \alpha \I_{ss_t} \sum_{k=t}^{T-1}(\mbox{$\gamma$}
 \mbox{$\lambda$}
)^{k-t}\delta_k.
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2334: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}% latex2html id marker 538
{1\over\mbox{$\alpha$}
 } \DVl_t(s_t)
   &=&\r^\lambda_t - V_t(s_t) \\ 
   &=& -V_t(s_t) +  (1-\mbox{$\lambda$}
 )\l^0[r_{t+1} + \mbox{$\gamma$}
  V_t(s_{t+1})]\\ 
   &&  \hspace*{.55in}   +~ (1-\mbox{$\lambda$}
 )\l^1[r_{t+1}+\mbox{$\gamma$}
  r_{t+2}+\g^2 V_t(s_{t+2})]\\ 
   &&  \hspace*{.55in}   +~ (1-\mbox{$\lambda$}
 )\l^2[r_{t+1}+\mbox{$\gamma$}
  r_{t+2}+\g^2 r_{t+3}+\g^3 V_t(s_{t+3})]\\ 
   &&  \hspace*{.55in}      
~~~~~~~~~~~~~~~~~~\vdots~~~~~~~~~\vdots~~~~~~~~~~~\vdots~~~~~~~~~~~~~~\ddots
\end{eqnarray*}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2350: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\l$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2352: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2354: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2356: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
 \mbox{$\lambda$}
 r_{t+2}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2358: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}% latex2html id marker 553
{1\over\mbox{$\alpha$}
 } \DVl_t(s_t)
   &=& -V_t(s_t)\\ 
   &&  +~ (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^0\left[r_{t+1} + \mbox{$\gamma$}
  V_t(s_{t+1}) -\mbox{$\gamma$}
 \mbox{$\lambda$}
 V_t(s_{t+1})\right]\\ 
   &&  +~ (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^1\left[r_{t+2} + \mbox{$\gamma$}
  V_t(s_{t+2}) -\mbox{$\gamma$}
 \mbox{$\lambda$}
 V_t(s_{t+2})\right]\\ 
   &&  +~ (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^2\left[r_{t+3} + \mbox{$\gamma$}
  V_t(s_{t+3}) -\mbox{$\gamma$}
 \mbox{$\lambda$}
 V_t(s_{t+3})\right]\\ 
   &&  ~\vdots\\ 
   &=& ~~~(\mbox{$\gamma$}
 \mbox{$\lambda$}
)^0\left[r_{t+1} + \mbox{$\gamma$}
  V_t(s_{t+1}) - V_t(s_{t})\right]\\ 
   && +~(\mbox{$\gamma$}
 \mbox{$\lambda$}
)^1\left[r_{t+2} + \mbox{$\gamma$}
  V_t(s_{t+2}) - V_t(s_{t+1})\right]\\ 
   && +~(\mbox{$\gamma$}
 \mbox{$\lambda$}
)^2\left[r_{t+3} + \mbox{$\gamma$}
  V_t(s_{t+3}) - V_t(s_{t+2})\right]\\ 
   && ~\vdots\\ 
   &\approx& \sum_{k=t}^\infty (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^{k-t}\delta_k\\ 
   &\approx& \sum_{k=t}^{T-1} (\mbox{$\gamma$}
 \mbox{$\lambda$}
)^{k-t}\delta_k.
\end{eqnarray*}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2418: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2422: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2424: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 2281
\sum_{t=0}^{T-1} \DVl_t(s_t) \I_{ss_t} = 
  \sum_{t=0}^{T-1}\mbox{$\alpha$}
 \I_{ss_t} \sum_{k=t}^{T-1}(\mbox{$\lambda$}
 \mbox{$\gamma$}
)^{k-t}\delta_k,\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2432
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2432: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2434: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2436: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2438: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2440: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2442: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2444: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2448: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2452: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2482
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2483
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2454: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2456: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2458: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2460: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2462: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2464: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2466: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2466: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=TDl-walk-results.ps,rheight=2.5in,width=3.411in}}

\vspace*{-.1in}

\label{fig:TDl-walk-results}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2484
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2485
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2468: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2470: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta_t = r_{t+1} +\mbox{$\gamma$}
  V_t(s_{t+1}) - V_{t-1}(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2472: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2474: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 2282
\Delta V_t(s_t) = \alpha\Bigl[\r_t^\lambda - V_{t-1}(s_t)\Bigr],
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2476
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2476: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2478: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2490: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2492: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2494: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2496: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2498: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2500: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2502: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2504: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2508: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2510: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2512: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2514: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e_t(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2516: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 2486
Q_{t+1}(s,a) = Q_t(s,a) + \mbox{$\alpha$}
 \delta_t e_t(s,a),\qquad\mbox{for all~} s,a \end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{displaymath}\delta_t = r_{t+1} + \mbox{$\gamma$}
  Q_t(s_{t+1}, a_{t+1}) - Q_t(s_t,a_t) \end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:action-trace}
e_t(s,a) = \left\{\begin{array}{ll}
    \mbox{$\gamma$}
 \mbox{$\lambda$}
 e_{t-1}(s,a) + 1 & \mbox{if $s\=s_t$ and $a=a_t$};\\ 
    \mbox{$\gamma$}
 \mbox{$\lambda$}
 e_{t-1}(s,a)     & \mbox{otherwise.}\end{array} \right.\quad\mbox{for
all~} s,a
\end{equation}
}

{\newpage
\clearpage
\samepage \begin{figure}[tbp]

\centerline{\psfig{figure=sarsabackup.ps,rheight=3.1in,width=3.276in}}


\label{sarsabackup}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2534: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2536: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2538: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2540: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2542: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2544: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2546: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2548: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2550: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2552: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,a)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2554: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{for all~} s,a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2556: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2566: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2568: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2570: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2574: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta \gets r + \mbox{$\gamma$}
  Q({s^\prime}
 ,{a^\prime}
 ) - Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2576: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,a) \gets e(s,a) + 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2578: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2582
$Q(s,a) \gets Q(s,a) + \alpha\delta e(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2582: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,a) \gets \mbox{$\gamma$}
 \mbox{$\lambda$}
 e(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2584: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2586: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\gets \ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2588: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2592: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2592: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:sarsal-algorithm}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=sarsa-trace-example.ps,rheight=1.5in,width=4.9345in}}

  
  \label{fig:sarsa-trace-example}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2610
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2611
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2594: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2596: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2598: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2600: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =.9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2602: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2604: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2620: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2622: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2624: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2626: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2628: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t,
a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2630: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t,a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2636: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$n\ge 3$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2640: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2642: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2644: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2646: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2648: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2650: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2652: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2654: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2656: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2658: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2660: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(s_t,a_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2662: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1} + \mbox{$\gamma$}
  \max_a Q_t(s_{t+1},a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2664: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_{t+n}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2666: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}r_{t+1} + \mbox{$\gamma$}
  r_{t+2} + \cdots +
\g^{n-1}r_{t+n} + \g^{n}\max_aQ_t(s_{t+n},a),\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2670: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2672: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2672: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=Watkins-Ql-backup.ps,rheight=3.2in,width=4.4395in}}


\label{fig:Watkins-Ql-backup}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2674: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2676: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2678: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}e_t(s,a) = \I_{ss_t} \cdot \I_{aa_t} + \left\{
  \begin{array}{ll}
    \mbox{$\gamma$}
 \mbox{$\lambda$}
\, e_{t-1}(s,a)        & \mbox{if $Q_{t-1}(s_t,a_t)=\max_a Q_{t-1}(s_t,a)$};\\ 
    0                          & \mbox{otherwise,}\\ 
  \end{array} \right.
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\I_{xy}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2690: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 2614
Q_{t+1}(s,a) = Q_t(s,a) + \mbox{$\alpha$}
 \, \delta_t\, e_t(s,a), \end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{displaymath}\delta_t = r_{t+1} + \mbox{$\gamma$}
  \max_{a^\prime} Q_t(s_{t+1}, a^\prime) - Q_t(s_t,a_t).
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2702: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,a)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2704: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2716: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2718: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2720: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2724: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a^* \gets \arg\max_b Q({s^\prime}
 ,b)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2726: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2728: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a^*\gets\ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2730: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta \gets r + \mbox{$\gamma$}
  Q({s^\prime}
 ,a^*) - Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2732: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,a) \gets e(s,a) + 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2734: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2738
$Q(s,a) \gets Q(s,a) + \alpha\delta e(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2738: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a'=a^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2740: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,a) \gets \mbox{$\gamma$}
 \mbox{$\lambda$}
 e(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2742: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,a) \gets 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2744: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2746: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\gets \ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2748: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2752: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2752: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:Watkins-Ql-alg}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2754: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2756: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2758: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2760: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2762: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2764: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2764: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}[tbp]

\centerline{\psfig{figure=Peng-Ql-backup.ps,rheight=3.0in,width=2.743in}}


\label{fig:Peng-Ql-backup}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2766: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2768: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2770: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2772: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2774: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2776: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2778: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2780: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2782: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2784: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2786: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2788: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2790: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2792: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2802: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2804: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 2798
p_{t+1}(s,a) = \left\{\begin{array}{ll}
  p_t(s,a) + \mbox{$\alpha$}
 \,\delta_t & \mbox{if $a\=a_t$ and $s\=s_t$}\\ 
  p_t(s,a)                & \mbox{otherwise,}
\end{array}\right.
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2812: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2814: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2816: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 728
\label{eq:ACtraceupdate}
  p_{t+1}(s,a) = p_t(s,a) + \alpha\, \delta_t\, e_t(s,a),
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2824: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2830: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 2799
p_{t+1}(s,a) = \left\{\begin{array}{ll}
  p_t(s,a) + \mbox{$\alpha$}
 \,\delta_t [1-\pi_t(s,a)]& \mbox{if $a\=a_t$ and
$s\=s_t$}\\ 
  p_t(s,a)                & \mbox{otherwise.}
\end{array}\right.
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\pi_t(s_t,a_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2838: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}e_t(s,a) = \left\{\begin{array}{ll}
    \mbox{$\gamma$}
 \mbox{$\lambda$}
 e_{t-1}(s,a) + 1-\pi_t(s_t,a_t) & \mbox{if $s\=s_t$ and
$a=a_t$};\\ 
    \mbox{$\gamma$}
 \mbox{$\lambda$}
 e_{t-1}(s,a)     & \mbox{otherwise,}\end{array} \right.
\end{equation}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=Traces.ps,rheight=1.55in,width=4.095in}}

\vspace*{-.1in}

\label{fig:Traces}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{equation}\label{Rtrace}
e_t(s) = \left\{\begin{array}{ll}
  \gamma\lambda e_{t-1}(s)      & \mbox{if $s\!\not=\!s_t$};\\ 
  1                         & \mbox{if $s\=s_t$}.\end{array} \right.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2864: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2866: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2868
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2868: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2870: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2866: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2868
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2868: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2870: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=walklambda2.ps,rheight=2.2in,width=2.674in}}

\vspace*{-.1in}

\label{fig:walklambda2}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2916
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2917
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,\mbox{\tt wrong\/})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2874: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(s,
\mbox{\tt right\/})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2876: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2878: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=right.ps,rheight=.8in,width=4.041in}}

\vspace*{-.1in}

\label{fig:right}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{equation}e_t(s,a) = \left\{\begin{array}{ll}
    1 + \mbox{$\gamma$}
 \mbox{$\lambda$}
 e_{t-1}(s,a) & \mbox{if $s\=s_t$ and $a\=a_t$};\\ 
    0                     & \mbox{if $s\=s_t$ and $a\not=a_t$};\\ 
    \mbox{$\gamma$}
 \mbox{$\lambda$}
 e_{t-1}(s,a)     & \mbox{if $s\not=s_t$.}\end{array}
\right.\quad\mbox{for all~} s,a
\end{equation}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2918
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2919
\arabic{exercise}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2920
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2921
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2900: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2922
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2923
\arabic{exercise}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2924
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2925
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2902: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =.9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2904: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2906
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2906: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2926
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2927
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2908: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2928: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2930: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2932: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2934: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2936: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2938: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2944
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2945
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2940: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2942: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2948: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2950: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2952: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}e_{t}(s) = \left\{\begin{array}{ll}
  \gamma\,\lambda_t\, e_{t-1}(s)      & \mbox{if $s\!\not=\!s_t$};\\ 
  \gamma\,\lambda_t\, e_{t-1}(s) + 1  & \mbox{if $s\=s_t$},\end{array} \right.
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\l_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2958: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2960: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2964: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\l_t=\mbox{$\lambda$}
 (s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2966: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2968: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2970: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2972: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2974: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}\rl_t &=& \sum_{n=1}^\infty\r_t^{(n)}\,(1-\l_{t+n})\!\prod_{i=t+1}^{t+n-1}\!\l_i\\ 
        &=& \sum_{k=t+1}^{T-1} \r_t^{(k-t)}\,(1-\l_k)\!\prod_{i=t+1}^{k-1}\!\l_i 
            ~~+~~ \r_t\! \prod_{i=t+1}^{T-1} \!\l_i.
\end{eqnarray*}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2980
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2981
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2976: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2978: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2982: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2984: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2986: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 3014
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 3015
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2990: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2994: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2996: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 3016
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 3017
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2998: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 3018
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 3019
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 3020
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 3021
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline3000: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 3022
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 3023
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline3002: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline3004: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$0<\mbox{$\lambda$}
 <1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline3006: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 3024
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 3025
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 3026
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 3027
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 3028
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 3029
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline3008: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline3010: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline3012: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{chapter}

\end{document}
