\batchmode
\documentstyle[12pt]{book}
\makeatletter


 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
%













\newcounter{bibliosection}

  


\newcounter{example}


\newcounter{exercise}





  





  






















































































\input psfig   %








\def\CE#1#2{{{E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpi#1#2{{{E_\pi}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpip#1#2{{{E_{\pi^\prime}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpistar#1#2{{{E_{\pi^*}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEggpistar#1#2{{{E_{\pi^*}}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEggpi#1#2{{{E_\pi}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEgpi#1#2{{{E_\pi}\!\Bigl\{#1\Bigm|#2\Bigr\}}}

\def\hatCE#1#2{{\hat {E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEgg#1#2{{{E}\biggl\{#1\biggm|#2\biggr\}}}

\def\E#1{{{E}\left\{#1\right\}}}

\def\Epi#1{{{E_\pi}\!\left\{#1\right\}}}

\def\Eggpi#1{{{E_\pi}\!\biggl\{#1\biggr\}}}

\def\Epip#1{{{E_{\pi^\prime}}\!\left\{#1\right\}}}

\def\Ea#1{{{E_a}\!\left\{#1\right\}}}

\def\Egg#1{{{E}\biggl\{#1\biggr\}}}

\def\CP#1#2{{{Pr}\left\{{#1}{\mid}{#2}\right\}}}

\def\Pr#1{{{Pr}\left\{#1\right\}}}

\def\w{{\bf w}}

\def\cross{\times}

\def\grad{\nabla}

\def\first#1#2{{\partial #1\over \partial #2}}

\def\second#1#2{{\partial^2 #1\over \partial #2^2}}

\def\implies{\Rightarrow}

\def\union{\cup}

\def\bibliosection#1{%
  \addtocounter{bibliosection}{1}%
  \medskip\goodbreak
  {\bf \arabic{chapter}.\arabic{bibliosection} #1}
  \smallskip}

\def\skipbibliosection{\addtocounter{bibliosection}{1}}

\def\example{
  \addtocounter{example}{1}
  \smallskip
  \noindent
  {\bf Example \arabic{chapter}.\arabic{example}}~~}

\def\exercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}}~~ }

\def\pexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}~(programming)~~}}

\def\starredexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}*~}}

\def\lastexample{\arabic{chapter}.\arabic{example}}

\def\lastexercise{\arabic{chapter}.\arabic{exercise}}

\def\resetmycounters{
  \setcounter{bibliosection}{0}
  \setcounter{example}{0}
  \setcounter{exercise}{0}}

\def\myfootnote#1{\footnote{#1}}

\def\lambdatarget{7.3}

\def\polefigure{3.2}

\def\StateAggregation{8.4}

\def\banditchap{2}

\def\probchap{3}

\def\DPchap{4}

\def\MCchap{5}

\def\TDchap{6}

\def\ETchap{7}

\def\FAchap{8}

\def\modelchap{9}

\def\CSchap{11}

\def\Sumchap{10}

\def\finalchap{10}

\def\eqincN{\banditchap.4}

\def\eqAverage{\banditchap.1}

\def\eqSAconditions{\banditchap.7}

\def\secinc{\banditchap.5}

\def\secnonstationary{\banditchap.6}

\def\softmaxsection{\banditchap.3}

\def\RCsection{2.8}

\def\predictionbackups{\probchap.4}

\def\DPbackupVpi{\probchap.4a}

\def\undiscountedsection{\probchap.?}

\def\maxbackups{\probchap.7}

\def\BellmanVpi{\probchap.10}

\def\PImprov{\DPchap.2}

\def\GPIsec{\DPchap.6}

\def\EstimateOtherPolicySec{\MCchap.5}

\def\TDprediction{\TDchap.2}

\def\simpleTDrule{\TDchap.2}

\def\randomwalkexample{\TDchap.2}

\def\randomwalkfigure{\TDchap.4}

\def\sarsasection{\TDchap.3}

\def\ACsection{\TDchap.6}

\def\Rlearningsection{\TDchap.7}

\def\afterstatesection{\TDchap.8}

\def\ACrule{\TDchap.8}

\def\ACruleplus{\TDchap.9}

\def\TileCodingSection{\FAchap.3.2}

\def\sarsalalg{\FAchap.8}

\def\lambdacomparison{\FAchap.10}

\def\heuristicsearchsection{\modelchap.7}

\def\planspacesection{\modelchap.10}

\def\uniformvsonpolicysection{\modelchap.9}

\def\ZDcasestudy{\CSchap.6}

\def\samuelsection{\CSchap.2}

\def\TDGammonSection{\CSchap.1}

\def\g{\mbox{$\gamma$}}

\def\a{\mbox{$\alpha$}}

\def\l{\mbox{$\lambda$}}

\def\e{\mbox{$\epsilon$}}

\def\ee{\mbox{$\varepsilon$}}

\def\n{\mbox{$n$}}

\def\r{\mbox{$R$}}

\def\rl{R^{\lambda}}

\def\sp{{s^\prime}}

\def\ap{{a^\prime}}

\def\TDl{TD($\lambda$)}

\def\itTDl{{\it TD\/}\mbox{\rm ($\lambda$)}}

\def\Ql{Q($\lambda$)}

\def\itQl{{\it Q\/}\mbox{\rm ($\lambda$)}}

\def\sarsal{Sarsa($\lambda$)}

\def\itsarsal{{\it Sarsa\/}\mbox{\rm ($\lambda$)}}

\def\={\!=\!}

\def\X{{\cal X}}

\def\A{{\cal A}}

\def\S{{\cal S}}

\def\P{{\cal P}}

\def\R{{\cal R}}

\def\Pssa{\P_{s\sp}^a}

\def\Rssa{\R_{s\sp}^a}

\def\endofexample{$\diamondsuit$}

\def\starredsection#1{\section{#1\ (*)}}

\def\&{and}

\def\figbody#1#2#3#4#5{%








\centerline{\psfig{figure=#1.ps,rheight=#3,#5}}}

\def\comment#1{}

\def\boxit#1{\vbox{\hrule\hbox{\vrule\kern3pt
             \vbox{\kern3pt#1\kern3pt}\kern3pt\vrule}\hrule}}

\def\myboxit#1{\vbox{\hrule\hbox{\vrule\kern6pt
             \vbox{\kern-17pt#1\kern-12pt}\kern8pt\vrule}\hrule}}

\def\sttright{{\mbox{\scriptsize\tt right}}}

\def\p#1{{\pi_#1}}

\def\pip{{\pi^\prime}}

\def\pipp{{\pi^{\prime\prime}}}

\def\mapright#1{\,\mathop{\longrightarrow}\limits^{#1}\,}

\def\raPE{\mapright{\mbox{\tiny E}}}

\def\raPI{\mapright{\mbox{\tiny I}}}

\def\policystable{\mbox{\it policy-stable}}

\makeatother
\newenvironment{tex2html_wrap}{}{}
\newwrite\lthtmlwrite
\def\lthtmltypeout#1{{\let\protect\string\immediate\write\lthtmlwrite{#1}}}%
\newbox\sizebox
\begin{document}
\pagestyle{empty}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1281: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mathop {\string\prm\space arg}\nolimits \mathop {\string\prm\space max}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1283: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p=0.4$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1285: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\setcounter{bibliosection}{0}
\setcounter{example}{0}
\setcounter{exercise}{0}
\setcounter{chapter}{3}
\stepcounter{chapter}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1287: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1289: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1291: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\P^a_{s{s^\prime}
 }={{Pr}\left\{{s_{t+1}\!=\!
 {s^\prime}
}{\mid}{s_t\=s,a_t\=a}\right\}}
$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1293: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R^a_{s{s^\prime}
 }={{E}\left\{{r_{t+1}}{\mid}{a_t\=a,s_t\=s,s_{t+1}={s^\prime}
 }\right\}}
$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1295: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1297: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1299: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${s^\prime}
 \in\S^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1301: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1303: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1305: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1307: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1309: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:BellmanV*}
V^*(s) &=& \max_a {{E}\left\{{{r_{t+1}+\mbox{$\gamma$}
  V^*(s_{t+1})} }{\mid}{ s_t\=s,a_t\=a}\right\}}
\nonumber\\ 
       &=& \max_a \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[\R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
  V^*({s^\prime}
 )\right],
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:BellmanQ*}
Q^*(s,a)&=&{{E}\biggl\{{r_{t+1}+\mbox{$\gamma$}
 \max_{a^\prime}
 
Q^*(s_{t+1},{a^\prime}
 )}\biggm|s_t\=s,a_t\=a\biggr\}}
\nonumber\\ 
   &=& \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[\R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
   \max_{a^\prime}
 
             Q^*({s^\prime}
 ,{a^\prime}
 )\right], 
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1319: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1321: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${s^\prime}
 \in\S^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1323: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1325: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1327: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1329: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:Vpi2}
V^\pi(s) &=& {{E_\pi}\!\left\{{ r_{t+1}+\mbox{$\gamma$}
  r_{t+2}+\g^2 r_{t+3}+ \cdots}{\mid}{s_t\=s}\right\}}
\nonumber\\ 
         &=& {{E_\pi}\!\left\{{{r_{t+1}+\mbox{$\gamma$}
  V^\pi(s_{t+1})} }{\mid}{ s_t\=s}\right\}}
  \\ 
\label{eq:VpiP}
         &=& \sum_{a} \pi(s,a) \sum_{s^\prime}
  \P^a_{s{s^\prime}
 } \left[\R^a_{s{s^\prime}
 } +
              \gamma V^\pi({s^\prime}
 ) \right],
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1335: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1341: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1343: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1345: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1347: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma<1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1349: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1351: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$|\S|$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1353: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$|\S|$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1355: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1357: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \in \S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1359: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_0, V_1, V_2, \ldots$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1361: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1363: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Re$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1365: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1367: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1369: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:Vk}
V_{k+1}(s) &=& {{E_\pi}\!\left\{{{r_{t+1}+\gamma V_k(s_{t+1})}}{\mid}{s_t\=s}\right\}}
\nonumber\\ 
           &=& \sum_{a} \pi(s,a) \sum_{s^\prime}
  \P^a_{s{s^\prime}
 } \left[\R^a_{s{s^\prime}
 } +
              \gamma V_k({s^\prime}
 ) \right],
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1371: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_k=V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1373: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1375: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{V_k\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1377: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1379: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$k\rightarrow\infty$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1381: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1383: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_{k+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1385: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1387: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_{k+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1395: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_k(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1397: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_{k+1}(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1399: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^{\pi}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1401: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\max_{s\in \S} |
V_{k+1}(s)-V_k(s)|$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1403: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1405: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1407: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1409: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta \gets 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1411: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1413: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v \gets V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1415: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s) \gets \sum_a \pi(s,a)\sum_{s^\prime}
  \P_{s{s^\prime}
 }^a 
                 \left[\R^a_{s{s^\prime}
 } + \mbox{$\gamma$}
  V({s^\prime}
 )\right]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1417: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta \gets \max(\Delta,|v-V(s)|)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1419: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1421
$\Delta<\theta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1421: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V\approx V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1423: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:policy-evaluation}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1471
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1472
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$4\times 4$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1425: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=4grid.ps,rheight=1.2in,height=1.2in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\S=\{1,2,\ldots,14\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1427: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\cal A}
 =\{\mbox{{\tt up}, {\tt down}, {\tt
right}, {\tt left}}\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1429: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\P^\sttright_{5,6}=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1431: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\P^\sttright_{5,10}=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1433: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\P^\sttright_{7,7}=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1435: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\R_{s{s^\prime}
 }^a=-1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1439: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s,\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1441: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{V_k\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1445: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1447: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1449: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=4gridconv.ps,rheight=5.85in,height=5.85in}}


\label{fig:4gridconv}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1473
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1474
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1451: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(11, \mbox{\tt down})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1453: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(7, \mbox{\tt down})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1455: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1475
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1476
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(15)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1457: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(15)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1459: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1477
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1478
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1461: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_0, Q_1, Q_2,
\ldots$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1463: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1479
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1480
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1465: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1483: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1485: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\not=\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1489: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1493: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1499: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:improv-single}
Q^\pi(s,a) &=& {{E_\pi}\!\left\{{{r_{t+1}+\gamma V^\pi(s_{t+1})}}{\mid}{s_t\=s,a_t\=a}\right\}}
 \\ 
           &=& \sum_{s^\prime}
  \P^a_{s{s^\prime}
 } \left[\R_{s{s^\prime}
 }^a+\gamma V^\pi({s^\prime}
 )\right].
\nonumber
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1501: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1507: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1509: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1515: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1517: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1519: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:PI-condition}
Q^\pi(s,{\pi^\prime} (s)) %
                 \ge V^\pi(s).
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1521: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1523: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1525: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:PI-conclusion}
 V^{\pi^\prime} \!(s) \ge V^\pi(s). 
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1527: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1529: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1531: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\pi^\prime} (s)=a\not=\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1533: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s,a) > V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1537: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1539: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1541: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^{\pi^\prime} \!(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1543: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}V^\pi(s) 
 &\le& Q^\pi(s,{\pi^\prime} (s)) \\ 
 &\le& {{E_{\pi^\prime}}\!\left\{{r_{t+1} + \gamma V^\pi(s_{t+1})}{\mid}{s_t\=s}\right\}}
 \\ 
 &\le& {{E_{\pi^\prime}}\!\left\{{r_{t+1} + \gamma Q^\pi(s_{t+1},{\pi^\prime} (s_{t+1}))}{\mid}{s_t\=s}\right\}}
 \\ 
 &\le& {{E_{\pi^\prime}}\!\left\{{r_{t+1} + \gamma{{E_{\pi^\prime}}\!\left\{{r_{t+2}+\gamma V^\pi(s_{t+2})}\right\}}
}{\mid}{s_t\=s}\right\}}
\\ 
 &\le& {{E_{\pi^\prime}}\!\left\{{r_{t+1} + \gamma r_{t+2} + \gamma^2 V^\pi(s_{t+2})}{\mid}{s_t\=s}\right\}}
\\ 
 &\le& {{E_{\pi^\prime}}\!\left\{{r_{t+1} + \gamma r_{t+2} + \gamma^2 r_{t+3}
          + \gamma^3 V^\pi(s_{t+3})}{\mid}{s_t\=s}\right\}}
\\ 
 &\vdots&\\ 
 &\le& {{E_{\pi^\prime}}\!\left\{{r_{t+1} + \gamma r_{t+2} + \gamma^2 r_{t+3} + \gamma^3 r_{t+4}
          + \cdots}{\mid}{s_t\=s}\right\}}
\\ 
 &\le& V^{\pi^\prime} \!(s).
\end{eqnarray*}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1545: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1547: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:improv}
{\pi^\prime} (s) &=& \arg\max_{a} Q^\pi(s,a)\nonumber\\ 
        &=& \arg \max_{a} {{E}\left\{{{r_{t+1}+\mbox{$\gamma$}
 
																																V^\pi(s_{t+1})}}{\mid}{s_t\=s,a_t\=a}\right\}}
\\ 
        &=& \arg \max_{a} \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[\R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
 
																																V^\pi({s^\prime}
 )
              																		\right],\nonumber
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\arg\max_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1553: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1557: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1559: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1561: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi = V^\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1563: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1565: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}V^{\pi^\prime} \!(s) &=& \max_a {{E}\left\{{{r_{t+1}+\mbox{$\gamma$}
  V^{\pi^\prime} \!(s_{t+1})}}{\mid}{s_t\=s,a_t\=a}\right\}}
\\ 
             &=& \max_a \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[ \R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
  V^{\pi^\prime} \!({s^\prime}
 )
                                          \right].
\end{eqnarray*}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1571: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1573: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1575: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1577: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1579: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1581: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}Q^\pi(s, {\pi^\prime} (s)) = \sum_a {\pi^\prime} (s,a)Q^\pi(s,a). \end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1587: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1589: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1591: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1593: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1595: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1597: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^{\pi^\prime} \!(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1599: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1607: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1609: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^{\pi^\prime} \!(s)\ge V^\pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1613: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1615: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1617: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1621: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1623: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1625: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pip$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1627: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pipp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1629: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\label{eq:PIseq}
\p0 \,\mathop{\longrightarrow}\limits^{\mbox{\tiny E}}\,  V^{\p0} \,\mathop{\longrightarrow}\limits^{\mbox{\tiny I}}\,  \p1 \,\mathop{\longrightarrow}\limits^{\mbox{\tiny E}}\,  V^{\p1} \,\mathop{\longrightarrow}\limits^{\mbox{\tiny I}}\,  \p2 \,\mathop{\longrightarrow}\limits^{\mbox{\tiny E}}\,  %
    \,\cdots\, \,\mathop{\longrightarrow}\limits^{\mbox{\tiny I}}\,  \pi^* \,\mathop{\longrightarrow}\limits^{\mbox{\tiny E}}\,  V^*,
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\raPE$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1631: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\raPI$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1633: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)\in\Re$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1635: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi(s)\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1637: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1639: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta \gets 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1641: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1643: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v \gets V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1645: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s) \gets \sum_{s^\prime} \P_{ss^\prime}^{\pi(s)} \left[\R^{\pi(s)}_{ss^\prime} + \gamma V(s^\prime)\right]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1647: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta \gets \max(\Delta,|v-V(s)|)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1649: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1651
$\Delta<\theta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1651: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{\it policy-stable}  \gets true$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1653: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1655: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$b \gets \pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1657: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi(s) \gets \arg\max_a \sum_{s^\prime} \P_{s{s^\prime} }^a
  \left[\R_{s{s^\prime} }^a
  + \gamma V(s^\prime)\right]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1659: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$b\not= \pi(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1661: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{\it policy-stable}  \gets false$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1663: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1665: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\arg\max$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1667: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1665: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\arg\max$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1667: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:policy-iteration}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1695
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1696
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${\lambda^n\over n!}e^{-\lambda}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1671: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1673: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1675: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\gamma$}
 =.9$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1677: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1679: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=jacks.ps,rheight=3.55in,width=5.12in}}


\label{fig:jacks}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1697
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1698
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1699
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1700
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1681: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1683: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1701
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1702
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1685: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon / |{\cal A}
 (s)|$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1689: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1691: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1703: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:value_iteration}
V_{k+1}(s) &=& \max_a {{E}\left\{{{r_{t+1}+\mbox{$\gamma$}
  V_k(s_{t+1})}}{\mid}{s_t\=s,a_t\=a}\right\}}
\\ 
           &=& \max_a \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[\R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
 
               V_k({s^\prime}
 )\right],\nonumber
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1709: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1711: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{V_k\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1713: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1715: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1717: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1719: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1721: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1723: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1727: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S^+$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1729: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta \gets 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1731: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1733: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$v \gets V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1735: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s) \gets \max_a \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[\R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
  V({s^\prime}
 )
                       \right]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1737: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\Delta \gets \max(\Delta,|v-V(s)|)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1739: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1741
$\Delta<\theta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1741: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1743: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi(s) = \arg\max_a \sum_{s^\prime}
  \P_{s{s^\prime}
 }^a \left[\R_{s{s^\prime}
 }^a + \mbox{$\gamma$}
  V({s^\prime}
 ) \right]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1745: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:value-iteration}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1773
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1774
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\{1,2,\ldots,99\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1747: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in\{0,1,\ldots,\min(s, 100-s)\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1749: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p=.4$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1757: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1759: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p=0.4$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1761: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p=0.4$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1761: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=gambler.ps,rheight=4.942in,width=4.746in}}


\label{fig:gambler}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1775
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1776
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1777
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1778
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p=.25$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1763: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p=.55$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1765: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1767
$\theta\rightarrow 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1767: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1779
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1780
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_{k+1}(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1769: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$10^{20}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1781: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1783: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$0 \le \gamma < 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1787: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1789: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\{s_k\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1791: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=GPI.ps,rheight=2.54in,width=1.8in}}


\label{fig:GPI}
\end{figure}
}

{\newpage
\clearpage
\samepage \psfig{figure=coconv.ps,rheight=1.6in,height=1.5in}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$m^n$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1805: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1807: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1809: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1811: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1813: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1815
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1816
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1817
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1818
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1819
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1820
\arabic{bibliosection}
}

\stepcounter{chapter}

\end{document}
