\batchmode
\documentstyle[12pt]{book}
\makeatletter


 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
%













\newcounter{bibliosection}

  


\newcounter{example}


\newcounter{exercise}





  





  






















































































\input psfig   %








\def\CE#1#2{{{E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpi#1#2{{{E_\pi}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpip#1#2{{{E_{\pi^\prime}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpistar#1#2{{{E_{\pi^*}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEggpistar#1#2{{{E_{\pi^*}}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEggpi#1#2{{{E_\pi}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEgpi#1#2{{{E_\pi}\!\Bigl\{#1\Bigm|#2\Bigr\}}}

\def\hatCE#1#2{{\hat {E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEgg#1#2{{{E}\biggl\{#1\biggm|#2\biggr\}}}

\def\E#1{{{E}\left\{#1\right\}}}

\def\Epi#1{{{E_\pi}\!\left\{#1\right\}}}

\def\Eggpi#1{{{E_\pi}\!\biggl\{#1\biggr\}}}

\def\Epip#1{{{E_{\pi^\prime}}\!\left\{#1\right\}}}

\def\Ea#1{{{E_a}\!\left\{#1\right\}}}

\def\Egg#1{{{E}\biggl\{#1\biggr\}}}

\def\CP#1#2{{{Pr}\left\{{#1}{\mid}{#2}\right\}}}

\def\Pr#1{{{Pr}\left\{#1\right\}}}

\def\w{{\bf w}}

\def\cross{\times}

\def\grad{\nabla}

\def\first#1#2{{\partial #1\over \partial #2}}

\def\second#1#2{{\partial^2 #1\over \partial #2^2}}

\def\implies{\Rightarrow}

\def\union{\cup}

\def\bibliosection#1{%
  \addtocounter{bibliosection}{1}%
  \medskip\goodbreak
  {\bf \arabic{chapter}.\arabic{bibliosection} #1}
  \smallskip}

\def\skipbibliosection{\addtocounter{bibliosection}{1}}

\def\example{
  \addtocounter{example}{1}
  \smallskip
  \noindent
  {\bf Example \arabic{chapter}.\arabic{example}}~~}

\def\exercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}}~~ }

\def\pexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}~(programming)~~}}

\def\starredexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}*~}}

\def\lastexample{\arabic{chapter}.\arabic{example}}

\def\lastexercise{\arabic{chapter}.\arabic{exercise}}

\def\resetmycounters{
  \setcounter{bibliosection}{0}
  \setcounter{example}{0}
  \setcounter{exercise}{0}}

\def\myfootnote#1{\footnote{#1}}

\def\lambdatarget{7.3}

\def\polefigure{3.2}

\def\StateAggregation{8.4}

\def\banditchap{2}

\def\probchap{3}

\def\DPchap{4}

\def\MCchap{5}

\def\TDchap{6}

\def\ETchap{7}

\def\FAchap{8}

\def\modelchap{9}

\def\CSchap{11}

\def\Sumchap{10}

\def\finalchap{10}

\def\eqincN{\banditchap.4}

\def\eqAverage{\banditchap.1}

\def\eqSAconditions{\banditchap.7}

\def\secinc{\banditchap.5}

\def\secnonstationary{\banditchap.6}

\def\softmaxsection{\banditchap.3}

\def\RCsection{2.8}

\def\predictionbackups{\probchap.4}

\def\DPbackupVpi{\probchap.4a}

\def\undiscountedsection{\probchap.?}

\def\maxbackups{\probchap.7}

\def\BellmanVpi{\probchap.10}

\def\PImprov{\DPchap.2}

\def\GPIsec{\DPchap.6}

\def\EstimateOtherPolicySec{\MCchap.5}

\def\TDprediction{\TDchap.2}

\def\simpleTDrule{\TDchap.2}

\def\randomwalkexample{\TDchap.2}

\def\randomwalkfigure{\TDchap.4}

\def\sarsasection{\TDchap.3}

\def\ACsection{\TDchap.6}

\def\Rlearningsection{\TDchap.7}

\def\afterstatesection{\TDchap.8}

\def\ACrule{\TDchap.8}

\def\ACruleplus{\TDchap.9}

\def\TileCodingSection{\FAchap.3.2}

\def\sarsalalg{\FAchap.8}

\def\lambdacomparison{\FAchap.10}

\def\heuristicsearchsection{\modelchap.7}

\def\planspacesection{\modelchap.10}

\def\uniformvsonpolicysection{\modelchap.9}

\def\ZDcasestudy{\CSchap.6}

\def\samuelsection{\CSchap.2}

\def\TDGammonSection{\CSchap.1}

\def\g{\mbox{$\gamma$}}

\def\a{\mbox{$\alpha$}}

\def\l{\mbox{$\lambda$}}

\def\e{\mbox{$\epsilon$}}

\def\ee{\mbox{$\varepsilon$}}

\def\n{\mbox{$n$}}

\def\r{\mbox{$R$}}

\def\rl{R^{\lambda}}

\def\sp{{s^\prime}}

\def\ap{{a^\prime}}

\def\TDl{TD($\lambda$)}

\def\itTDl{{\it TD\/}\mbox{\rm ($\lambda$)}}

\def\Ql{Q($\lambda$)}

\def\itQl{{\it Q\/}\mbox{\rm ($\lambda$)}}

\def\sarsal{Sarsa($\lambda$)}

\def\itsarsal{{\it Sarsa\/}\mbox{\rm ($\lambda$)}}

\def\={\!=\!}

\def\X{{\cal X}}

\def\A{{\cal A}}

\def\S{{\cal S}}

\def\P{{\cal P}}

\def\R{{\cal R}}

\def\Pssa{\P_{s\sp}^a}

\def\Rssa{\R_{s\sp}^a}

\def\endofexample{$\diamondsuit$}

\def\starredsection#1{\section{#1\ (*)}}

\def\&{and}

\def\figbody#1#2#3#4#5{%








\centerline{\psfig{figure=#1.ps,rheight=#3,#5}}}

\def\comment#1{}

\def\boxit#1{\vbox{\hrule\hbox{\vrule\kern3pt
             \vbox{\kern3pt#1\kern3pt}\kern3pt\vrule}\hrule}}

\def\myboxit#1{\vbox{\hrule\hbox{\vrule\kern6pt
             \vbox{\kern-17pt#1\kern-12pt}\kern8pt\vrule}\hrule}}

\def\rela{\tilde}

\makeatother
\newenvironment{tex2html_wrap}{}{}
\newwrite\lthtmlwrite
\def\lthtmltypeout#1{{\let\protect\string\immediate\write\lthtmlwrite{#1}}}%
\newbox\sizebox
\begin{document}
\pagestyle{empty}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1305: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1307
$\unhbox \voidb@x \hbox {$\alpha $}=0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1307: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1309
$\alpha $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1309: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1311
$\alpha $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1311: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1313
$\alpha $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1313: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1315: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1317
$\alpha $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1317: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1319: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1321: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$2.73$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1323: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\setcounter{bibliosection}{0}
\setcounter{example}{0}
\setcounter{exercise}{0}
\setcounter{chapter}{5}
\stepcounter{chapter}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1325: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1327: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1329: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1331: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1335: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1337: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1341: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1343: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 270
\label{MCrule}
V(s_t) \gets V(s_t) + \mbox{$\alpha$}
  \Bigl[\r_t - V(s_t)\Bigr],
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1347: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1351
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1351: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1353
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1353: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1355: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$R_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1357: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1361: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1363: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 277
\label{TD0}
V(s_t) \gets V(s_t) + \mbox{$\alpha$}
  \Bigl[r_{t+1} + \gamma V(s_{t+1}) - V(s_t)\Bigr].
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1367: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{t+1} + \gamma V_t(s_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1369: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{ExpReturn}
V^\pi(s) &=& {{E_\pi}\!\left\{{\r_t}{\mid}{s_t\=s}\right\}}
 \\ 
           &=& {{E_\pi}\!\biggl\{\sum_{k=0}^\infty \gamma^k r_{t+k+1}\biggm|s_t\=s\biggr\}}
 \nonumber\\ 
           &=& {{E_\pi}\!\biggl\{r_{t+1} 
          +\gamma \sum_{k=0}^\infty \gamma^k r_{t+k+2}\biggm|s_t\=s\biggr\}}
\nonumber\\ 
\label{BellmanRight}
         &=& {{E_\pi}\!\biggl\{{r_{t+1} + \gamma V^\pi(s_{t+1})}\biggm|s_t\=s\biggr\}}
.
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(s_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1371: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t(s_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1373: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1375: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1377: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=TDbackup.ps,rheight=0.8in,width=.2in}}

  
  \label{fig:TDbackup}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1379: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1381: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \mbox{action given by $\pi$ for $s$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1385: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1391: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1393
$V(s) \gets V(s) + \alpha\left[r+\gamma V({s^\prime}
 ) - V(s)\right]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1393: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1395: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1399: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1399: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:TD0}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1419
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1420
\arabic{example}
}

{\newpage
\clearpage
\samepage \begin{tabular}{lccc}
                 &{\it elapsed time}&  {\it predicted}   & {\it predicted}\\ 
\medskip
{\it situation}  &{\it (minutes)}&{\it time-to-go}& {\it total time}\\ 
leaving office, friday at 6 &  0   & 30    & 30  \\ 
reach car, raining          &  5   & 35    & 40  \\ 
exiting highway             & 20   & 15    & 35  \\ 
2ndary road, behind truck   & 30   & 10    & 40  \\ 
entering home street        & 40   &  3    & 43  \\ 
arrive home                 & 43   &  0    & 43
\end{tabular}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1401: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=DriveMC.ps,rheight=2.1in,width=3.1in}}

\vspace*{-.1in}

\label{DriveMC}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1403
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1403: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\r_t-V_t(s_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1405: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1407
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1407: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1/2$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1409: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1411
$\mbox{$\alpha$}
 =1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1411: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=DriveTD.ps,rheight=2.1in,width=3.33in}}

\vspace*{-.2in}

\label{DriveTD}
\end{figure}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1421: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1423: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1479
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1480
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1425
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1425: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi(C)=0.5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1433: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${1\over 6},
{2\over 6}, {3\over 6}, {4\over 6}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1439: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${5\over 6}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1441: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1443
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1443: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1445
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1445: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)=0.5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1447: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1451: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=random-walk.ps,rheight=0.6in,width=4.25in}}


\label{fig:random-walk}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1453
$\mbox{$\alpha$}
 =0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1453: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1453
$\mbox{$\alpha$}
 =0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1453: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=sample-run.ps,rheight=2.41in,width=3.46in}}


\label{fig:sample-run}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1455
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1455: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1457
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1457: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1455
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1455: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1457
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1457: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=walk-results.ps,rheight=2.67in,width=3.99in}}


\label{fig:walk-results}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1481
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1482
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1483
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1484
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(A)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1459: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1485
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1486
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1461
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1461: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1487
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1488
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1463
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1463: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1489
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1490
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${1\over 6},
{2\over 6}, {3\over 6}, {4\over 6}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1465: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${5\over 6}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1467: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1497
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1497: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1499
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1499: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1501
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1501: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1585
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1586
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1503
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1503: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1505
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1505: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1507
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1507: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1509: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1511: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1513
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1513: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1513
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1513: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=batch-walk.ps,rheight=2.4in,width=3.44in}}


\label{fig:batch-walk}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1515: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1587
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1588
\arabic{example}
}

{\newpage
\clearpage
\samepage \begin{displaymath}\begin{array}{ll}
A,0,B,0\hspace{1.5in}&B,1\\ B,1&B,1\\ B,1&B,1\\ B,1&B,0
\end{array}\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(A)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1527: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(B)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1529: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(B)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1531: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${3\over 4}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1533: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(A)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1539: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${3\over 4}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1547: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${3\over 4}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1551: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=AB0-example.ps,rheight=1in,width=1.53in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(A)={3\over 4}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1553: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V(A)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1557: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1561: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1571
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1571: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$N^2$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1575: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$N^3$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1577: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1589: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1591: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1597: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1599: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=beads.ps,rheight=.35in,width=3.67in}
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 462
\label{SarsaRule}
Q(s_t, a_t) \gets Q(s_t, a_t) + \mbox{$\alpha$}
  \Bigl[r_{t+1} + 
                         \mbox{$\gamma$}
  Q(s_{t+1}, a_{t+1}) - Q(s_t,a_t) \Bigr].
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1605: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1607: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s_{t+1},a_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1609: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(s_t, a_t, r_{t+1}, s_{t+1}, a_{t+1})$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1611: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1613: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1615: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1617: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1619: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1621: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1631: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1637: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1639: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1641: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1645: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1647
$Q(s,a) \gets Q(s,a) + \alpha\bigl[r+\gamma Q({s^\prime}
 ,{a^\prime}
 )
                                        - Q(s,a)\bigr]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1647: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1649: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\gets \ap$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1651: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:SarsaBox}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1657: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1659: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1661: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =1/t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1663: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1687
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1688
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1667: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1669: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1671
$\mbox{$\alpha$}
 =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1671: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1673: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1677: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1679: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=windy.ps,rheight=1.83in,width=4.068in}}


\label{fig:windy}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=windy-graph.ps,rheight=1.925in,width=3.445in}}


\label{fig:windy-graph}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1689
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1690
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1691
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1692
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1693
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1694
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 516
\label{eq:Q-learning}
Q(s_t,a_t) \gets Q(s_t,a_t) + \mbox{$\alpha$}
 \Bigl[
      r_{t+1}+\mbox{$\gamma$}
 \max_a Q(s_{t+1},a)-Q(s_t,a_t)\Bigr].
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1701: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1703: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1705: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1707: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1717: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1723: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1725
$Q(s,a) \gets Q(s,a) + \alpha\Bigl[r+\mbox{$\gamma$}
  \max_{{a^\prime}
 } Q({s^\prime}
 ,{a^\prime}
 )
                                        - Q(s,a)\Bigr]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1725: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1727: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:QBox}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1731: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1731: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=cliff.ps,rheight=4in,width=4.319in}}


\label{fig:cliff}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=Qbackup.ps,rheight=0.8in,width=0.59in}}

  
  \label{fig:Qbackup}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1751
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1752
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1737: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1739: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1741: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1743: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1745: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1753
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1754
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1755
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1756
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \begin{eqnarray*}% latex2html id marker 554
\label{Q-learning-exp}
Q(s_t,a_t) &\gets& Q(s_t,a_t) + \alpha\Bigl[
      r_{t+1}+\gamma\,{{E}\left\{{{Q(s_{t+1},a_{t+1})}}{\mid}{s_t}\right\}}
-Q(s_t,a_t)\Bigr]\\ 
 &\gets& Q(s_t,a_t) + \alpha\Bigl[
      r_{t+1}+\gamma\sum_a \pi(s_t,a) Q(s_{t+1},a)-Q(s_t,a_t)\Bigr]
\end{eqnarray*}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=actor-critic.ps,rheight=2.9in,width=2.9475in}}


\label{fig:actor-critic}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{displaymath}\delta_t = r_{t+1} + \mbox{$\gamma$}
  V(s_{t+1}) - V(s_t),  \end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1769: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1771: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1773: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}%

\pi_t(s,a) = {{Pr}\left\{{a_{t}\=a}{\mid}{s_t\=s}\right\}}
= {e^{p(s,a)} \over \sum_b
e^{p(s,b)} }.
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1775: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p(s_t,a_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1783: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}%
p(s_t,a_t) \gets p(s_t,a_t) + \beta \delta_t, 
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1785: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1787: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1789: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}%
 p(s_t,a_t) \gets p(s_t,a_t) + \beta \delta_t \left[1-\pi_t(s_t,a_t)\right].
\end{displaymath}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1799: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1801: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\rho^\pi = \lim_{n\rightarrow\infty} {1  \over n}\sum_{t=1}^n {{E_\pi}\!\left\{r_t\right\}}
, \end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1803: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\tilde  V^\pi(s) = \sum_{k=1}^\infty {{E_\pi}\!\left\{{r_{t+k} - \rho^\pi }{\mid}{ s_t\=s}\right\}}
,  \end{displaymath}
}

{\newpage
\clearpage
\samepage \begin{displaymath}\tilde  Q^\pi(s,a) = \sum_{k=1}^\infty {{E_\pi}\!\left\{{r_{t+k}-\rho^\pi }{\mid}{
s_t\=s,a_t\=a}\right\}}
.\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1805: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1807: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1809: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1811: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tilde  Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1815: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1817: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1819: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1821: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1823: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \mbox{current state}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1827: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1833: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1839: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1841
$Q(s,a) \gets Q(s,a) + \mbox{$\alpha$}
 ~\bigl[r-\rho+ \max_{{a^\prime}
 } Q({s^\prime}
 ,{a^\prime}
 ) - Q(s,a)\bigr]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1841: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)=\max_a Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1843: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho \gets \rho + \beta~\bigl[r-\rho+\max_{{a^\prime}
 } Q({s^\prime}
 ,{a^\prime}
 ) - \max_a
Q(s,a)\bigr]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1845: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1847
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1847: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1849: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1847
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1847: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1849: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}

\label{fig:R-learning}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1879
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1880
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$h=.5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1859: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p=.06$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1861: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1863
$\mbox{$\alpha$}
 =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1863: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta=.01$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1865: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon=.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1867: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1869: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1871: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1873: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$2.73$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1875: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\rho$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1873: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$2.73$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1875: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=access.ps,rheight=3.19in,width=3.655in}}

  
  \label{fig:access}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1881
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1882
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \psfig{figure=TTTafter.ps,rheight=1.8in,width=3in}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1885
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1886
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1887: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1889
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1890
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1891
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1892
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1893
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1894
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1895
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1896
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1897
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1898
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1899
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1900
\arabic{bibliosection}
}

\stepcounter{chapter}

\end{document}
