\batchmode
\documentstyle[12pt]{book}
\makeatletter


 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
%













\newcounter{bibliosection}

  


\newcounter{example}


\newcounter{exercise}





  





  






















































































\input psfig   %








\def\CE#1#2{{{E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpi#1#2{{{E_\pi}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpip#1#2{{{E_{\pi^\prime}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpistar#1#2{{{E_{\pi^*}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEggpistar#1#2{{{E_{\pi^*}}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEggpi#1#2{{{E_\pi}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEgpi#1#2{{{E_\pi}\!\Bigl\{#1\Bigm|#2\Bigr\}}}

\def\hatCE#1#2{{\hat {E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEgg#1#2{{{E}\biggl\{#1\biggm|#2\biggr\}}}

\def\E#1{{{E}\left\{#1\right\}}}

\def\Epi#1{{{E_\pi}\!\left\{#1\right\}}}

\def\Eggpi#1{{{E_\pi}\!\biggl\{#1\biggr\}}}

\def\Epip#1{{{E_{\pi^\prime}}\!\left\{#1\right\}}}

\def\Ea#1{{{E_a}\!\left\{#1\right\}}}

\def\Egg#1{{{E}\biggl\{#1\biggr\}}}

\def\CP#1#2{{{Pr}\left\{{#1}{\mid}{#2}\right\}}}

\def\Pr#1{{{Pr}\left\{#1\right\}}}

\def\w{{\bf w}}

\def\cross{\times}

\def\grad{\nabla}

\def\first#1#2{{\partial #1\over \partial #2}}

\def\second#1#2{{\partial^2 #1\over \partial #2^2}}

\def\implies{\Rightarrow}

\def\union{\cup}

\def\bibliosection#1{%
  \addtocounter{bibliosection}{1}%
  \medskip\goodbreak
  {\bf \arabic{chapter}.\arabic{bibliosection} #1}
  \smallskip}

\def\skipbibliosection{\addtocounter{bibliosection}{1}}

\def\example{
  \addtocounter{example}{1}
  \smallskip
  \noindent
  {\bf Example \arabic{chapter}.\arabic{example}}~~}

\def\exercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}}~~ }

\def\pexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}~(programming)~~}}

\def\starredexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}*~}}

\def\lastexample{\arabic{chapter}.\arabic{example}}

\def\lastexercise{\arabic{chapter}.\arabic{exercise}}

\def\resetmycounters{
  \setcounter{bibliosection}{0}
  \setcounter{example}{0}
  \setcounter{exercise}{0}}

\def\myfootnote#1{\footnote{#1}}

\def\lambdatarget{7.3}

\def\polefigure{3.2}

\def\StateAggregation{8.4}

\def\banditchap{2}

\def\probchap{3}

\def\DPchap{4}

\def\MCchap{5}

\def\TDchap{6}

\def\ETchap{7}

\def\FAchap{8}

\def\modelchap{9}

\def\CSchap{11}

\def\Sumchap{10}

\def\finalchap{10}

\def\eqincN{\banditchap.4}

\def\eqAverage{\banditchap.1}

\def\eqSAconditions{\banditchap.7}

\def\secinc{\banditchap.5}

\def\secnonstationary{\banditchap.6}

\def\softmaxsection{\banditchap.3}

\def\RCsection{2.8}

\def\predictionbackups{\probchap.4}

\def\DPbackupVpi{\probchap.4a}

\def\undiscountedsection{\probchap.?}

\def\maxbackups{\probchap.7}

\def\BellmanVpi{\probchap.10}

\def\PImprov{\DPchap.2}

\def\GPIsec{\DPchap.6}

\def\EstimateOtherPolicySec{\MCchap.5}

\def\TDprediction{\TDchap.2}

\def\simpleTDrule{\TDchap.2}

\def\randomwalkexample{\TDchap.2}

\def\randomwalkfigure{\TDchap.4}

\def\sarsasection{\TDchap.3}

\def\ACsection{\TDchap.6}

\def\Rlearningsection{\TDchap.7}

\def\afterstatesection{\TDchap.8}

\def\ACrule{\TDchap.8}

\def\ACruleplus{\TDchap.9}

\def\TileCodingSection{\FAchap.3.2}

\def\sarsalalg{\FAchap.8}

\def\lambdacomparison{\FAchap.10}

\def\heuristicsearchsection{\modelchap.7}

\def\planspacesection{\modelchap.10}

\def\uniformvsonpolicysection{\modelchap.9}

\def\ZDcasestudy{\CSchap.6}

\def\samuelsection{\CSchap.2}

\def\TDGammonSection{\CSchap.1}

\def\g{\mbox{$\gamma$}}

\def\a{\mbox{$\alpha$}}

\def\l{\mbox{$\lambda$}}

\def\e{\mbox{$\epsilon$}}

\def\ee{\mbox{$\varepsilon$}}

\def\n{\mbox{$n$}}

\def\r{\mbox{$R$}}

\def\rl{R^{\lambda}}

\def\sp{{s^\prime}}

\def\ap{{a^\prime}}

\def\TDl{TD($\lambda$)}

\def\itTDl{{\it TD\/}\mbox{\rm ($\lambda$)}}

\def\Ql{Q($\lambda$)}

\def\itQl{{\it Q\/}\mbox{\rm ($\lambda$)}}

\def\sarsal{Sarsa($\lambda$)}

\def\itsarsal{{\it Sarsa\/}\mbox{\rm ($\lambda$)}}

\def\={\!=\!}

\def\X{{\cal X}}

\def\A{{\cal A}}

\def\S{{\cal S}}

\def\P{{\cal P}}

\def\R{{\cal R}}

\def\Pssa{\P_{s\sp}^a}

\def\Rssa{\R_{s\sp}^a}

\def\endofexample{$\diamondsuit$}

\def\starredsection#1{\section{#1\ (*)}}

\def\&{and}

\def\figbody#1#2#3#4#5{%








\centerline{\psfig{figure=#1.ps,rheight=#3,#5}}}

\def\comment#1{}

\def\w{{\vec\theta}}

\def\wcomp{\theta}

\def\F{{\cal F}}

\def\leadsto{\rightarrow}

\def\arrow{\rightarrow}

\def\boxit#1{\vbox{\hrule\hbox{\vrule\kern3pt
             \vbox{\kern3pt#1\kern3pt}\kern3pt\vrule}\hrule}}

\def\myboxit#1{\vbox{\hrule\hbox{\vrule\kern6pt
             \vbox{\kern-17pt#1\kern-12pt}\kern8pt\vrule}\hrule}}

\def\refBairdexample{8.4}

\makeatother
\newenvironment{tex2html_wrap}{}{}
\newwrite\lthtmlwrite
\def\lthtmltypeout#1{{\let\protect\string\immediate\write\lthtmlwrite{#1}}}%
\newbox\sizebox
\begin{document}
\pagestyle{empty}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1463: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1465: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1469
${\mathaccent "017E \theta }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1469: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1475: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1477: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1479: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1481: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-\mathop {\string\prm\space max}_a Q_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1483: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1485
$\alpha $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1485: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1487: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$9 \times 9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1489: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\unhbox \voidb@x \hbox {$\gamma $}=.99$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1491: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1493
$\unhbox \voidb@x \hbox {$\alpha $}=0.01$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1493: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1495
${\mathaccent "017E \theta }_0=(1,1,1,1,1,10,1)^T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1495: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1497: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1499: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1501: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\setcounter{bibliosection}{0}
\setcounter{example}{0}
\setcounter{exercise}{0}
\setcounter{chapter}{7}
\stepcounter{chapter}
\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1503: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1505: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1509: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1511: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1513: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1515: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1517: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1519: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1521: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1523: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1525: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1527: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1529: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\rightarrow  v$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1531: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \rightarrow 
{{E_\pi}\!\left\{{r_{t+1}+\mbox{$\gamma$}
  V_t(s_{t+1})}{\mid}{s_t\=s}\right\}}
$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1539: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t
\rightarrow  \r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1541: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t \rightarrow  r_{t+1}+\mbox{$\gamma$}
  V_t(s_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1543: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1545: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t \rightarrow  \r^\lambda_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1547: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1551: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \rightarrow  v$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1553: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \rightarrow  v$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1563: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1565: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1567: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1571: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1573: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1575: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:MSE}
MSE(\w_t) = \sum_{s\in\S} P(s) \Bigl( V^\pi(s)-V_t(s) \Bigr)^2,
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1579: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1587: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1589: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w^{*}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1591: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1593
$MSE(\w^{*})\le MSE({\vec\theta} )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1593: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1595: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1597: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1599
$MSE(\w^{*})\le MSE({\vec\theta} )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1599: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1601: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1603: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t=(\wcomp_t(1), \wcomp_t(2), \ldots, \wcomp_t(n))^T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1605: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1609: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1611: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1613: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t \rightarrow  V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1617: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1619: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1621: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1623: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}% latex2html id marker 293
\w_{t+1} &=& \w_t -{1\over 2} \mbox{$\alpha$}
  \nabla_{\w_t}\Bigl(V^\pi(s_t)-V_t(s_t)\Bigr)^2
\nonumber\\ 
\label{eq:GD}
         &=& \w_t + \mbox{$\alpha$}
  \Bigl(V^\pi(s_t)-V_t(s_t)\Bigr)\nabla_{\w_t} V_t(s_t),
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1631
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1631: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\nabla_{\w_t} f(\w_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1633: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$({\partial f(\w_t)\over \partial \wcomp_t(1)}
,
{\partial f(\w_t)\over \partial \wcomp_t(2)}
, \ldots, {\partial f(\w_t)\over \partial \wcomp_t(2)}
)^T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1637: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1641: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1643: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1645: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t \rightarrow  v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1649: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1651: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1653: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1655: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1657: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1659: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1661: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 311
\label{eq:approxGD}
\w_{t+1} = \w_t + \mbox{$\alpha$}
  \Bigl(v_t-V_t(s_t)\Bigr)\nabla_{\w_t} V_t(s_t).
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1665: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${{E}\left\{v_t\right\}}
=V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1667: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1671: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1673
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1673: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1675: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$R_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1677: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1679: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t=R_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1681: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1683: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1685: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1689: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1691: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1693: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t=\r^\lambda_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1695: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1697: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 319
\label{eq:TDlforward}
\w_{t+1} = \w_t + \mbox{$\alpha$}
  \left(R^\lambda_t-V_t(s_t)\right) \nabla_{\w_t} V_t(s_t).
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 <1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1701: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r^\lambda_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1703: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1705: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t=E_\pi\!\left\{r_{t+1}+\mbox{$\gamma$}
  V_t(s_{t+1})\mid s_t\right\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1707: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1709: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1711: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 330
\label{eq:TDl2}
\w_{t+1} = \w_t + \alpha\,\delta_t\, \vec e_{t},
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1713: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\delta_t=r_{t+1}+\mbox{$\gamma$}
  V_t(s_{t+1})-V_t(s_t), 
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e_{t}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1717: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1719: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:TDltrace}
\vec e_{t} = \mbox{$\gamma$}
 \mbox{$\lambda$}
 \vec e_{t-1} + \nabla_{\w_t}V_t(s_t),
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e_0=\vec 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1725: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1727: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1729: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1731: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \mbox{initial state of episode}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1733: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \mbox{action given by $\pi$ for $s$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1735: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1741: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta \gets r + \mbox{$\gamma$}
  V({s^\prime}
 ) - V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1743: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1745
$\vec e \gets \mbox{$\gamma$}
 \mbox{$\lambda$}
\vec e + \nabla_{{\vec\theta} } V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1745: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1747
${\vec\theta}  \gets {\vec\theta}  + \mbox{$\alpha$}
 \,\delta\, \vec e$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1747: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1749: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1753: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1755: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1759: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1753: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1755: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1759: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:GDTDl}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1773
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1774
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1761: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1763: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1775
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1776
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1777
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1778
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1765: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1767: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1771: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1779
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1780
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1783: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1785: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec\phi_s = (\phi_s(1),
\phi_s(2),\ldots,\phi_s(n))^T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1789: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1791: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:linearV}
 V_t(s) = \w_t^T \vec\phi_s 
          = \sum_{i=1}^n \wcomp_t(i) \phi_s(i).
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1793: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\nabla_{\w_t} V_t(s) = \vec\phi_s. \end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1795: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1797: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1799: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_\infty$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1801: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:MSEbound}
MSE(\w_\infty) ~~\le~~  {1-\mbox{$\gamma$}
 \mbox{$\lambda$}
 \over 1-\mbox{$\gamma$}
 }\,  MSE(\w^*).
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\gamma\lambda \over 1-\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1809: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1811: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1817
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1818
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1819
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1820
\arabic{exercise}
}

\stepcounter{subsection}
{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=CoarseCoding.ps,rheight=1.9in,width=2.226in}}


\label{fig:CoarseCoding}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1829: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=generalization.ps,rheight=1.7in,width=5.2in}}


\label{fig:generalization}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1851
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1852
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1837: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1839
$\mbox{$\alpha$}
 ={0.2\over m}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1839: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1843: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=acuity.ps,rheight=3in,width=4.944in}}


\label{fig:acuity}
\end{figure}
}

\stepcounter{subsection}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1855
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1855: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1857
$\mbox{$\alpha$}
 ={1\over m}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1857: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t \rightarrow  v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1861: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1863: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_{t+1}(s_t) = v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1865: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1867
$\mbox{$\alpha$}
 ={1\over 10m}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1867: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1875
$\nabla_{{\vec\theta} }V_t(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1875: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=2Dgrid.ps,rheight=1.3in,width=1.212in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\times$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1885: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=CMAC.ps,rheight=1.5in,width=4.71in}}


\label{fig:CMAC}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=tilings.ps,rheight=1.37in,width=3.94in}}


\label{fig:tilings}
\end{figure}
}

{\newpage
\clearpage
\samepage \psfig{figure=hash-tile.ps,rheight=1.3635in,width=1.3635in}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1895
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1896
\arabic{exercise}
}

\stepcounter{subsection}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$[0,1]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1899: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\phi_s(i)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1903: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$c_i$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1907: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sigma_i$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1909: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\phi_s(i) = \exp\left(-{||s-c_i||^2\over 2\sigma_i^2}\right). \end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{figure}[tb]

\centerline{\psfig{figure=RBFs.ps,rheight=1in,width=3.078in}}


\label{RBFs}
\end{figure}
}

\stepcounter{subsection}
\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t \approx Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1925: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1927: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t \rightarrow  v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1929: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t,a_t \rightarrow  v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1931: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1933: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s_t,a_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1935: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$R_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1937: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}+\mbox{$\gamma$}
  Q_t(s_{t+1},a_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1939: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 1913
\w_{t+1} = \w_t + \mbox{$\alpha$}
  \Bigl(v_t-Q_t(s_t,a_t)\Bigr)\nabla_{\w_t} Q_t(s_t,a_t).
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1943: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 1914
%

\w_{t+1} = \w_t + \alpha\,\delta_t\, \vec e_{t},
\end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{displaymath}\delta_t=r_{t+1}+\mbox{$\gamma$}
  Q_t(s_{t+1},a_{t+1})-Q_t(s_t,a_t),\end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{displaymath}%

\vec e_{t} = \mbox{$\gamma$}
 \mbox{$\lambda$}
 \vec e_{t-1} + \nabla_{\w_t}Q_t(s_t,a_t),
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e_0=\vec 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1951: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1953: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1955: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1959: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(s_t,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1961: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a^*_t=\arg\max_a Q_t(s_t,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1963: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1965: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1967: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e=\vec 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1969: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\gets\mbox{initial state of episode}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1971: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1973: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\F_a \gets \mbox{set of features present in $s,a$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1975: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1977
$Q_a \gets \sum_{i\in\F_a} \theta (i)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1977: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \arg\max_a Q_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1979: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1981: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \mbox{a random action $\in{\cal A}
 (s)$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1983: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e \gets \mbox{$\gamma$}
 \mbox{$\lambda$}
\vec e$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1985: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\bar a\not=a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1987: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$i\in\F_{\bar a}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1989: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(i) \gets 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1991: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$i\in\F_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1993: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(i) \gets e(i) + 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1995: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(i) \gets 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1997: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2003: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta \gets r - Q_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2005: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2007: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\F_a \gets \mbox{set of features present in ${s^\prime}
 ,a$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2009: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2011
$Q_a \gets \sum_{i\in\F_a} \theta (i)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2011: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${a^\prime}
  \gets \arg\max_a Q_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2013: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2015: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${a^\prime}
  \gets \mbox{a random action $\in{\cal A}
 (s)$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2017: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta \gets \delta + \mbox{$\gamma$}
  Q_\ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2019: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2021
${\vec\theta}  \gets {\vec\theta}  + \mbox{$\alpha$}
 \,\delta\, \vec e$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2021: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2023: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2025: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2027: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2029: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2027: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2029: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:FAsarsa}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2031: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e=\vec 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2033: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\gets\mbox{initial state of episode}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2035: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2037: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\F_a \gets \mbox{set of features present in $s,a$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2039: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2041
$Q_a \gets \sum_{i\in\F_a} \theta (i)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2041: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\e$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2043: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \arg\max_a Q_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2045: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e \gets \mbox{$\gamma$}
 \mbox{$\lambda$}
\vec e$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2047: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \mbox{a random action $\in{\cal A}
 (s)$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2049: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\vec e \gets 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2051: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$i\in\F_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2053: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$e(i) \gets e(i) + 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2055: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2061: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta \gets r - Q_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2063: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2065: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\F_a \gets \mbox{set of features present in ${s^\prime}
 ,a$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2067: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2069
$Q_a \gets \sum_{i\in\F_a} \theta (i)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2069: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${a^\prime}
  \gets \arg\max_a Q_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2071: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\delta \gets \delta + \mbox{$\gamma$}
  Q_\ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2073: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2075
${\vec\theta}  \gets {\vec\theta}  + \mbox{$\alpha$}
 \,\delta\, \vec e$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2075: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2077: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2079: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2081: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2079: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2081: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:FAQ}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2083: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2085: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2087: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\F_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2089: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\F_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2093: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2099: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2165
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2166
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-\max_a Q_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2111: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-\max_a Q_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2111: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=car-episode.ps,rheight=3.15in,width=5.06in}}


\label{fig:car-episode}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$x_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2121: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\dot x_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2123: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}x_{t+1} = bound \left[ x_t + \dot x_{t+1} \right]
\end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{displaymath}\dot x_{t+1} = bound \left[ \dot x_t + 0.001 a_t + -0.0025 \cos(3x_t) \right],
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-1.2 \le x_{t+1} \le 0.5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2127: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-0.07 \le
\dot x_{t+1} \le 0.07$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2129: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$x_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2131: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\dot x_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2133: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$9 \times 9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2135: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =.9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2137: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2139: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2141
$\mbox{$\alpha$}
 =.05$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2141: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${0.1\over m}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2143: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2145: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2149
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2149: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2151: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2153: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2155
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2155: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2157: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$9 \times 9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2159: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2155
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2155: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2157: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$9 \times 9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2159: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=car-results.ps,rheight=2.282in,width=4.774in}}


\label{fig:car-results}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2167
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2168
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2171: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 <1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2173: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2175: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2179: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2181: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2269
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2270
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2183: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2187: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{\vec\phi_s\,:\,s\in\S\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2189: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t=\vec 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2191: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=BairdExample.ps,rheight=1.876in,width=3.017in}}


\label{fig:BairdExample}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2193: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}% latex2html id marker 2169
\w_{k+1} = \w_k + \mbox{$\alpha$}
  \sum_s \Bigl({{E}\left\{{r_{t+1}+\mbox{$\gamma$}
 
V_t(s_{t+1})}{\mid}{s_t\=s}\right\}}
-V_k(s)\Bigr)\nabla_{\w_k} V_k(s).
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2201: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
 =.99$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2203: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2205
$\mbox{$\alpha$}
 =0.01$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2205: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_0=(1,1,1,1,1,10,1)^T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2207: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
 =.99$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2203: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2205
$\mbox{$\alpha$}
 =0.01$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2205: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_0=(1,1,1,1,1,10,1)^T$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2207: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=BairdBlowup.ps,rheight=2.4in,width=4.365in}}


\label{fig:BairdBlowup}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2209: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2211: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{\vec\phi_s\,:\,s\in\S\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2213: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=TVRexample.ps,rheight=1.25in,width=1.842in}}


\label{fig:TVRexample}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2271
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2272
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\wcomp_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2215: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\wcomp_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2217: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$2\wcomp_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2219: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\wcomp_k=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2221: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\wcomp_{k+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2223: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}% latex2html id marker 628
\wcomp_{k+1} &=& \arg\min_{\theta \in\Re} \sum_{s\in\S} 
    \Bigl( V_\theta (s) - {{E_\pi}\!\left\{{r_{t+1}+\mbox{$\gamma$}
 
V_{\wcomp_k}(s_{t+1})}{\mid}{s_t\=s}\right\}}
 \Bigr)^2\nonumber\\ 
         &=& \arg\min_{\theta \in\Re} 
     \bigl(\theta  - \mbox{$\gamma$}
  2\wcomp_k\bigr)^2 
         + \bigl(2\theta  - (1-\mbox{$\epsilon$}
 ) \mbox{$\gamma$}
  2\wcomp_k\bigr)^2 \nonumber\\ 
         &=& {6-4\mbox{$\epsilon$}
 \over 5} \mbox{$\gamma$}
  \wcomp_k,
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_\wcomp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2237: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\wcomp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2239: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{\wcomp_k\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2241: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
 > {5\over 6-4\mbox{$\epsilon$}
 }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2243: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\wcomp_0\not=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2245: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2247: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{BellmanError}
\sum_{s} P(s) \Bigl( {{E_\pi}\!\left\{{r_{t+1}+\mbox{$\gamma$}
  V_t(s_{t+1})}{\mid}{s_t\=s}\right\}}
 - V_t(s) \Bigr)^2.
\end{equation}
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}% latex2html id marker 648
\w_{t+1} &=& \w_t -{1\over 2} \mbox{$\alpha$}
  \nabla_{\w_t} 
    \Bigl( {{E_\pi}\!\left\{{r_{t+1}+\mbox{$\gamma$}
  V_t(s_{t+1})}\right\}}
 - V_t(s_t) \Bigr)^2 \\ 
&=&  \w_t+\mbox{$\alpha$}
  \Bigl( {{E_\pi}\!\left\{{r_{t+1}+\mbox{$\gamma$}
  V_t(s_{t+1})}\right\}}
 - V_t(s_t) \Bigr)
       \Bigl( \nabla_{\w_t}V_t(s_t)-{{E_\pi}\!\left\{{\nabla_{\w_t}V_t(s_{t+1})}\right\}}
\Bigr),
\end{eqnarray*}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2259: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2261: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2273
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2274
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2275: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2277: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2279: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2281: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2277: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2279: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2281: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=lambdacomparison.ps,rheight=3.96in,width=5.14in}}


\label{fig:lambdacomparison}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2283: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2285: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2289: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2293: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\lambda$}
 <1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2295: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\gamma\lambda\over 1-\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2297: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2319
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2320
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2321
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2322
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2299: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{\vec\phi_s\,:\,s\in\S\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2301: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\l$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2303: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\w_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2305: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2307: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2309: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2323
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2324
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2311: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2313: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2315: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2325
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2326
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$A_{R-P}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2317: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{chapter}

\end{document}
