\batchmode
\documentstyle[12pt]{book}
\makeatletter


 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
%













\newcounter{bibliosection}

  


\newcounter{example}


\newcounter{exercise}





  





  






















































































\input psfig   %








\def\CE#1#2{{{E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpi#1#2{{{E_\pi}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpip#1#2{{{E_{\pi^\prime}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpistar#1#2{{{E_{\pi^*}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEggpistar#1#2{{{E_{\pi^*}}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEggpi#1#2{{{E_\pi}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEgpi#1#2{{{E_\pi}\!\Bigl\{#1\Bigm|#2\Bigr\}}}

\def\hatCE#1#2{{\hat {E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEgg#1#2{{{E}\biggl\{#1\biggm|#2\biggr\}}}

\def\E#1{{{E}\left\{#1\right\}}}

\def\Epi#1{{{E_\pi}\!\left\{#1\right\}}}

\def\Eggpi#1{{{E_\pi}\!\biggl\{#1\biggr\}}}

\def\Epip#1{{{E_{\pi^\prime}}\!\left\{#1\right\}}}

\def\Ea#1{{{E_a}\!\left\{#1\right\}}}

\def\Egg#1{{{E}\biggl\{#1\biggr\}}}

\def\CP#1#2{{{Pr}\left\{{#1}{\mid}{#2}\right\}}}

\def\Pr#1{{{Pr}\left\{#1\right\}}}

\def\w{{\bf w}}

\def\cross{\times}

\def\grad{\nabla}

\def\first#1#2{{\partial #1\over \partial #2}}

\def\second#1#2{{\partial^2 #1\over \partial #2^2}}

\def\implies{\Rightarrow}

\def\union{\cup}

\def\bibliosection#1{%
  \addtocounter{bibliosection}{1}%
  \medskip\goodbreak
  {\bf \arabic{chapter}.\arabic{bibliosection} #1}
  \smallskip}

\def\skipbibliosection{\addtocounter{bibliosection}{1}}

\def\example{
  \addtocounter{example}{1}
  \smallskip
  \noindent
  {\bf Example \arabic{chapter}.\arabic{example}}~~}

\def\exercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}}~~ }

\def\pexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}~(programming)~~}}

\def\starredexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}*~}}

\def\lastexample{\arabic{chapter}.\arabic{example}}

\def\lastexercise{\arabic{chapter}.\arabic{exercise}}

\def\resetmycounters{
  \setcounter{bibliosection}{0}
  \setcounter{example}{0}
  \setcounter{exercise}{0}}

\def\myfootnote#1{\footnote{#1}}

\def\lambdatarget{7.3}

\def\polefigure{3.2}

\def\StateAggregation{8.4}

\def\banditchap{2}

\def\probchap{3}

\def\DPchap{4}

\def\MCchap{5}

\def\TDchap{6}

\def\ETchap{7}

\def\FAchap{8}

\def\modelchap{9}

\def\CSchap{11}

\def\Sumchap{10}

\def\finalchap{10}

\def\eqincN{\banditchap.4}

\def\eqAverage{\banditchap.1}

\def\eqSAconditions{\banditchap.7}

\def\secinc{\banditchap.5}

\def\secnonstationary{\banditchap.6}

\def\softmaxsection{\banditchap.3}

\def\RCsection{2.8}

\def\predictionbackups{\probchap.4}

\def\DPbackupVpi{\probchap.4a}

\def\undiscountedsection{\probchap.?}

\def\maxbackups{\probchap.7}

\def\BellmanVpi{\probchap.10}

\def\PImprov{\DPchap.2}

\def\GPIsec{\DPchap.6}

\def\EstimateOtherPolicySec{\MCchap.5}

\def\TDprediction{\TDchap.2}

\def\simpleTDrule{\TDchap.2}

\def\randomwalkexample{\TDchap.2}

\def\randomwalkfigure{\TDchap.4}

\def\sarsasection{\TDchap.3}

\def\ACsection{\TDchap.6}

\def\Rlearningsection{\TDchap.7}

\def\afterstatesection{\TDchap.8}

\def\ACrule{\TDchap.8}

\def\ACruleplus{\TDchap.9}

\def\TileCodingSection{\FAchap.3.2}

\def\sarsalalg{\FAchap.8}

\def\lambdacomparison{\FAchap.10}

\def\heuristicsearchsection{\modelchap.7}

\def\planspacesection{\modelchap.10}

\def\uniformvsonpolicysection{\modelchap.9}

\def\ZDcasestudy{\CSchap.6}

\def\samuelsection{\CSchap.2}

\def\TDGammonSection{\CSchap.1}

\def\g{\mbox{$\gamma$}}

\def\a{\mbox{$\alpha$}}

\def\l{\mbox{$\lambda$}}

\def\e{\mbox{$\epsilon$}}

\def\ee{\mbox{$\varepsilon$}}

\def\n{\mbox{$n$}}

\def\r{\mbox{$R$}}

\def\rl{R^{\lambda}}

\def\sp{{s^\prime}}

\def\ap{{a^\prime}}

\def\TDl{TD($\lambda$)}

\def\itTDl{{\it TD\/}\mbox{\rm ($\lambda$)}}

\def\Ql{Q($\lambda$)}

\def\itQl{{\it Q\/}\mbox{\rm ($\lambda$)}}

\def\sarsal{Sarsa($\lambda$)}

\def\itsarsal{{\it Sarsa\/}\mbox{\rm ($\lambda$)}}

\def\={\!=\!}

\def\X{{\cal X}}

\def\A{{\cal A}}

\def\S{{\cal S}}

\def\P{{\cal P}}

\def\R{{\cal R}}

\def\Pssa{\P_{s\sp}^a}

\def\Rssa{\R_{s\sp}^a}

\def\endofexample{$\diamondsuit$}

\def\starredsection#1{\section{#1\ (*)}}

\def\&{and}

\def\figbody#1#2#3#4#5{%








\centerline{\psfig{figure=#1.ps,rheight=#3,#5}}}

\def\comment#1{}

\def\sttsearch{{\mbox{\scriptsize\tt search}}}

\def\sttwait{{\mbox{\scriptsize\tt wait}}}

\def\Aprime{${\rm A}^\prime$}

\def\Bprime{${\rm B}^\prime$}

\def\sttputt{{\mbox{\scriptsize\tt putt}}}

\def\h{{\mbox{\tt h}}}

\def\ll{{\mbox{\tt l}}}

\def\s{{\mbox{\tt s}}}

\def\w{{\mbox{\tt w}}}

\def\re{{\mbox{\tt re}}}

\def\sh{{\mbox{\scriptsize\tt h}}}

\def\sll{{\mbox{\scriptsize\tt l}}}

\def\ss{{\mbox{\scriptsize\tt s}}}

\def\sw{{\mbox{\scriptsize\tt w}}}

\def\sre{{\mbox{\scriptsize\tt re}}}

\makeatother
\newenvironment{tex2html_wrap}{}{}
\newwrite\lthtmlwrite
\def\lthtmltypeout#1{{\let\protect\string\immediate\write\lthtmlwrite{#1}}}%
\newbox\sizebox
\begin{document}
\pagestyle{empty}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${s^\prime }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1662: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \in {\fam \tw@ A}(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1664: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1666: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1668: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1670: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1672: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\setcounter{bibliosection}{0}
\setcounter{example}{0}
\setcounter{exercise}{0}
\setcounter{chapter}{2}
\stepcounter{chapter}
\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$t=0,1,2,3,\ldots$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1674: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1678: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1680: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t\in{\cal A}
 (s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1682: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\cal A}
 (s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1684: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1686: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}\in \Re$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1688: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1690: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1692: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1696: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}[t]
  
\centerline{\psfig{figure=Agent-Env.ps,rheight=1.414in,width=4.03in}}

  \vskip -.1in
  
  \label{fig:interaction}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1698: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi_t(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1700: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t=a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1702: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t=s$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1704: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1716
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1717
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1706: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1718
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1719
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1710: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1720
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1721
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1712: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1722
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1723
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1724
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1725
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1726
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1727
\arabic{exercise}
}

\stepcounter{section}
\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}, r_{t+2}, r_{t+3}, \dots$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1740: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1742: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:return-total}
\r_t = r_{t+1} + r_{t+2} + r_{t+3} + \cdots + r_T,
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1746: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1748: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$T=\infty$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1750: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1754: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:return}
\r_t = r_{t+1} + \gamma r_{t+2} + \gamma^2 r_{t+3}  + \cdots
   & = & \sum_{k=0}^\infty \gamma^{k} r_{t+k+1},
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1756: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$0\le\gamma\le 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1758: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma^{k-1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1762: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma<1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1764: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{r_k\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1766: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1768: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1770: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1772: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1774: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}[bp]
  
\centerline{\psfig{figure=Cart-Pole.ps,rheight=1.4in,width=2.825in}}

  \vskip -.05in
  
  \label{fig:Cart-Pole}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1792
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1793
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-\gamma^k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1780: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1784: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1794
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1795
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1796
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1797
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1798: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_{t,i}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1802: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_{t,i}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1808: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t,i}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1810: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi_{t,i}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1812: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$T_{i}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1814: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1816: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_{t,i}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1818: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=absorbing-beads.ps,rheight=.58in,width=4.14in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1820: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$+1,+1,+1,0,0,0,\ldots$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1822: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\r_t = \sum_{k=0}^T \gamma^k r_{t+k+1},
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$T=\infty$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1828: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1830: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{equation}\label{eq:causal-dynamics}
{{Pr}\left\{{s_{t+1}={s^\prime}
 ,r_{t+1}=r|s_t,a_t,r_t,s_{t-1},a_{t-1},\ldots,r_1,s_0,a_0}{\mid}{,}\right\}}

\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1838: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t,a_t,r_t,\ldots,r_1,s_0,a_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1842: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:Markov-dynamics}
{{Pr}\left\{{s_{t+1}={s^\prime}
 ,r_{t+1}=r|s_t,a_t}{\mid}{,}\right\}}

\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1848: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1852: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1854: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1856: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t,a_t,r_t,\ldots,r_1,s_0,a_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1860: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1866
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1867
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1862: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1868
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1869
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1864: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1870
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1871
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1876: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:transition_probs}
 \P_{s{s^\prime}
 }^a = {{Pr}\left\{{s_{t+1}\!=\!
 {s^\prime}
}{\mid}{s_t\=s,a_t\=a}\right\}}
.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1882: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:exp_rewards}
\R_{s{s^\prime}
 }^a = {{E}\left\{{r_{t+1} }{\mid}{ s_t\=s,a_t\=a,s_{t+1}={s^\prime}
 }\right\}}
.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Pssa$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1884: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Rssa$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1886: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1990
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1991
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S=\{\mbox{\tt high}, \mbox{\tt low}\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1888: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}{\cal A}
 (\mbox{\tt high}) &=& \{\mbox{\tt search}, \,\mbox{\tt wait}\}\\ 
{\cal A}
 (\mbox{\tt low}) &=& \{ \mbox{\tt search}, \,\mbox{\tt wait}, \, \mbox{\tt
recharge}\}.
\end{eqnarray*}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1890
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1890: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1892
$1-\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1892: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1894: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1896: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R^\sttsearch$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1900: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R^\sttwait$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1902: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R^{\mbox{\scriptsize\tt search}}  > \R^\sttwait$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1904: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1958: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \in {\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1960: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1958: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \in {\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1960: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{table}% latex2html id marker 450
\begin{center}
\begin{tabular}{lll|l|l}
$s=s_t$    &  ${s^\prime}
 =s_{t+1}$ & $a=a_t$   & $\P_{s{s^\prime}
 }^a$ & $\R_{s{s^\prime}
 }^a$ \\ 
\hline {\tt high} & {\tt high} & {\tt search}  &  $\alpha$   & $\R^\sttsearch$\\  
{\tt high} & {\tt low} & {\tt search}   &	 $1-\alpha$ & $\R^\sttsearch$\\  
{\tt low} & {\tt high} & {\tt search}   &  $1-\beta$  & $-3$\\ 
{\tt low} & {\tt low} & {\tt search}    &  $\beta$    & $\R^\sttsearch$\\ 
{\tt high} & {\tt high} & {\tt wait}    &  $1$        & $\R^\sttwait$\\  
{\tt high} & {\tt low} & {\tt wait}     &  $0$        & $\R^\sttwait$\\ 
{\tt low} & {\tt high} & {\tt wait}     &  $0$        & $\R^\sttwait$\\ 
{\tt low} & {\tt low} & {\tt wait}      &  $1$        & $\R^\sttwait$\\ 
{\tt low} & {\tt high} & {\tt recharge} &  $1$        & $0$\\ 
{\tt low} & {\tt low} & {\tt recharge}  &  $0$        & $0$.
\end{tabular}
\end{center}

\label{table:dynamics}
\end{table}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1962: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1974: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1976: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(s, {s^\prime}
 , a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1978: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1980: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\P_{s{s^\prime}
 }^a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1982: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R_{s{s^\prime}
 }^a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1984: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=st-graph.ps,rheight=2.5in,width=4.6in}}

  
  \label{fig:st-graph}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1986: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1992
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1993
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1994: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in
\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1996: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1998: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2000: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2008: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2010: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2014: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2016: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:Vpi}
V^\pi(s) = {{E_\pi}\!\left\{{\r_t}{\mid}{s_t\=s}\right\}}
 = {{E_\pi}\!\biggl\{\sum_{k=0}^\infty \gamma^k r_{t+k+1} \biggm|
								s_t\=s\biggr\}}
,
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${{E_\pi}\!\left\{\right\}}
$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2018: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2020: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2024: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2026: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2032: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^{\pi}(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2034: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2040: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:Qpi}
  Q^\pi(s,a) = {{E_\pi}\!\left\{{\r_t}{\mid}{s_t\=s,a_t=a}\right\}}

    = {{E_\pi}\!\biggl\{\sum_{k=0}^\infty \gamma^k r_{t+k+1}\biggm|s_t\!=\!
  s, a_t\=a\biggr\}}
.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2042: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2044: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2046: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2048: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2050: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2052: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2054: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2056: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2058: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2060: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:Bellman-Vpi}
V^\pi(s) &=& {{E_\pi}\!\left\{{\r_t}{\mid}{s_t\=s}\right\}}
\nonumber\\ 
    &=& {{E_\pi}\!\biggl\{\sum_{k=0}^\infty\gamma^kr_{t+k+1}\biggm|s_t\=s\biggr\}}
\nonumber\\ 
    &=& {{E_\pi}\!\biggl\{r_{t+1} +\gamma\sum_{k=0}^\infty\gamma^kr_{t+k+2}\biggm|s_t\=s\biggr\}}
\nonumber\\ 
    &=& \sum_a \pi(s,a) \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[ \R_{s{s^\prime}
 }^a + \gamma 
          {{E_\pi}\!\biggl\{\sum_{k=0}^\infty \g^k r_{t+k+2} \biggm| s_{t+1}\!=\!
 {s^\prime}
\biggr\}}
 \right]\nonumber\\ 
    &=& \sum_a \pi(s,a) \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[ \R_{s{s^\prime}
 }^a + \gamma 
          V^\pi({s^\prime}
 ) \right],
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2068: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2070: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2072: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2074: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2076: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2080: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2086: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2088: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2090: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2092: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2090: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2092: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=prediction-backups.ps,rheight=1.5in,width=4.5in}}


\label{fig:prediction-backups}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2204
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2205
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\rm A}^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2098: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\rm B}^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2102: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}[htbp]
 

\centerline{\psfig{figure=Grid.ps,rheight=1.3in,width=3.73in}}

  
		\label{fig:grid}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2104: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma=0.9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2106: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\rm A}^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2108: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\rm B}^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2110: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\rm B}^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2112: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2114: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=golf.ps,rheight=5.088in,width=4.744in}}

  
		\label{fig:golf}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2206
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2207
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^{\mbox{\scriptsize\tt putt}} (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2116: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-\infty$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2130: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2132: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2208
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2209
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2134: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2136: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi({s^\prime}
 ,{a^\prime}
 )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2138: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2140: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2210
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2211
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2142: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$+0.7$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2144: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$+2.3$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2146: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$+0.4$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2148: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$-0.4$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2150: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$+0.7$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2152: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2212
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2213
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2162: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2214
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2215
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2216
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2217
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \psfig{figure=little-V-backup.ps,rheight=.8in,width=4.16in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2166: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2168: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t=s$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2170: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2172: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2174: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2218
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2219
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2176: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2178: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=little-Q-backup.ps,rheight=1in,width=2.55in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^{\pi}(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2180: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2182: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2184: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t\=s$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2186: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t\=a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2188: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\P_{s{s^\prime}
 }^a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2190: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R_{s{s^\prime}
 }^a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2192: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2222: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2224: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2226: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi\ge\pi^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2228: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s) \ge V^{\pi^\prime}(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2230: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2232: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2234: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2236: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:optimal-V}
V^*(s) = \max_{\pi} V^\pi(s), 
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2238: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2240: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:optimal-Q}
Q^*(s,a) = \max_{\pi} Q^\pi(s,a), 
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2242: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2244: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2246: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2252: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2254: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:optimal-Q-optimal-V}
Q^*(s,a)={{E}\left\{{r_{t+1}+\gamma V^*(s_{t+1}) }{\mid}{ s_t\=s, a_t\=a}\right\}}
.
\end{equation}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2410
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2411
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(s,\mbox{\tt driver\/})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2256: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(s,\mbox{\tt
driver\/})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2260: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2268: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2270: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2272: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2274: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}V^*(s) &=& \max_a Q^{\pi^*}(s,a)\nonumber\\ 
       &=& \max_a {{E_{\pi^*}}\!\biggl\{\r_t\biggm| s_t\=s,a_t\=a\biggr\}}
\nonumber\\ 
       &=& \max_{a \in {\cal A}
 (s)} {{E_{\pi^*}}\!\biggl\{ \sum_{k=0}^\infty \g^k r_{t+k+1} \biggm|
s_t\=s,a_t\=a\biggr\}}
\nonumber\\ 
       &=& \max_{a \in {\cal A}
 (s)} {{E_{\pi^*}}\!\biggl\{ r_{t+1} + \mbox{$\gamma$}
 \sum_{k=0}^\infty \g^k
r_{t+k+2} \biggm| s_t\=s,a_t\=a\biggr\}}
\nonumber\\ 
       &=& \max_{a \in {\cal A}
 (s)} {{E}\left\{{{r_{t+1} + \gamma V^*(s_{t+1})} }{\mid}{
																																																							s_t\=s,a_t\=a}\right\}}
\\ 
\label{eq:bellman-V*}
       &=& \max_{a \in {\cal A}
 (s)} \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[ \R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
  V^*({s^\prime}
 )
																																																							\right],
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2280: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2282: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}Q^*(s,a) &=& {{E}\biggl\{{ r_{t+1} + \gamma \max_{a^\prime}
    Q^* (s_{t+1}, a^\prime)} \biggm|s_t=s, a_t=a \biggr\}}
\\ 
    &=&  \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[\R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
  \max_{a^\prime}
  Q^*({s^\prime}
 ,{a^\prime}
 )\right]
\end{eqnarray*}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2286: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2288: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2290: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2292: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2294: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2296: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2294: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2296: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=max-backups.ps,rheight=1.5in,width=4.5in}}


\label{fig:max-backups}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R_{s{s^\prime}
 }^a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2304: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\P_{s{s^\prime}
 }^a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2306: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2308: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2310: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2312: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2316: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2318: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2320: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2322: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2324: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2326: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2328: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2332: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2412
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2413
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*({\mbox{\tt h}} )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2334: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}% latex2html id marker 712
V^*({\mbox{\tt h}} )&=&\max\left\{\begin{array}{l}
    \P_{{\mbox{\scriptsize\tt h}} {\mbox{\scriptsize\tt h}}}^\ss [\R_{{\mbox{\scriptsize\tt h}} {\mbox{\scriptsize\tt h}}}^\ss + \mbox{$\gamma$}
  V^*({\mbox{\tt h}} )] + \P_{{\mbox{\scriptsize\tt h}} {\mbox{\scriptsize\tt l}}}^{\mbox{\tt s}}  [\R_{{\mbox{\scriptsize\tt h}} {\mbox{\scriptsize\tt l}}}^\s+\mbox{$\gamma$}
  V^*({\mbox{\tt l}} )],\\  
   	\P_{{\mbox{\scriptsize\tt h}} {\mbox{\scriptsize\tt h}}}^{\mbox{\scriptsize\tt w}}  [\R_{{\mbox{\scriptsize\tt h}} {\mbox{\scriptsize\tt h}}}^{\mbox{\scriptsize\tt w}}  + \mbox{$\gamma$}
  V^*({\mbox{\tt h}} )]	+ \P_{{\mbox{\scriptsize\tt h}} {\mbox{\scriptsize\tt l}}}^{\mbox{\scriptsize\tt w}}  [\R_{{\mbox{\scriptsize\tt h}} {\mbox{\scriptsize\tt l}}}^\sw+\mbox{$\gamma$}
  V^*({\mbox{\tt l}} )]\\ 
																					\end{array}\right\}\\ 
       &=&\max\left\{\begin{array}{l}
    \alpha [\R^\ss+\mbox{$\gamma$}
  V^*({\mbox{\tt h}} )]+ (1-\alpha)[\R^\ss + \mbox{$\gamma$}
  V^*({\mbox{\tt h}} )],\\ 
				1 [ \R^\sw+\mbox{$\gamma$}
  V^*({\mbox{\tt h}} )]+0 [\R^\sw+\mbox{$\gamma$}
  V^*({\mbox{\tt l}} )]
																					\end{array}\right\}\\ 
       &=&\max\left\{\begin{array}{l}
					\R^\ss+\mbox{$\gamma$}
 [\alpha V^*({\mbox{\tt h}} )+(1-\alpha)V^*({\mbox{\tt l}} )],\\ 
					\R^{\mbox{\scriptsize\tt w}}  + \mbox{$\gamma$}
  V^*({\mbox{\tt h}} ) 
                     \end{array}\right\}.
\end{eqnarray*}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*({\mbox{\tt l}} )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2356: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}V^*({\mbox{\tt l}} )=\max\left\{\begin{array}{l}
      \beta \R^\ss -3(1-\beta) +\mbox{$\gamma$}
 [(1-\beta) V^*({\mbox{\tt h}} )+\beta V^*({\mbox{\tt l}} )]\\ 
      \R^{\mbox{\scriptsize\tt w}}   +\mbox{$\gamma$}
  V^*({\mbox{\tt l}} ),\\ 
      \mbox{$\gamma$}
  V^*({\mbox{\tt h}} )
                          \end{array}\right\}.
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R^\ss$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2364: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R^\sw$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2366: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2368
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2368: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2370: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2372: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$0 \le \gamma < 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2374: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 2376
$0 \le \alpha, \beta \le 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2376: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*({\mbox{\tt h}} )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2378: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*({\mbox{\tt l}} )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2380: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2414
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2415
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2382: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\rm A}^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2386: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\rm B}^\prime$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2390: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}[htbp]
  
\centerline{\psfig{figure=Grid-Solution.ps,rheight=1.4in,width=4.097in}}

  
		\label{fig:grid-solution}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$10^{20}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2392: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2394: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2396: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2398: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\rm
A}^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2400: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2416
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2417
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2418
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2419
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(s,\mbox{\tt putter\/})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2402: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2420
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2421
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2404: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2422
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2423
\arabic{exercise}
}

\stepcounter{section}
\stepcounter{section}
\stepcounter{section}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2448
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2449
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2450
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2451
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2452
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2453
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2454
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2455
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Rssa$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2426: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R_s^a={{E}\left\{{r_{t+1}}{\mid}{s_t=s,a_t=a}\right\}}
$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2428: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Rssa$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2430: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\R_s^a=\sum_{{s^\prime}
 }\P_{s{s^\prime}
 }^a \R_{s{s^\prime}
 }^a.\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Rssa$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2432: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$R_s^a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2434: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Rssa$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2436: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 2456
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 2457
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2438: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$h(P,M)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2440: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline2446: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{chapter}

\end{document}
