\batchmode
\documentstyle[12pt]{book}
\makeatletter


 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
%













\newcounter{bibliosection}

  


\newcounter{example}


\newcounter{exercise}





  





  






















































































\input psfig   %








\def\CE#1#2{{{E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpi#1#2{{{E_\pi}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpip#1#2{{{E_{\pi^\prime}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpistar#1#2{{{E_{\pi^*}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEggpistar#1#2{{{E_{\pi^*}}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEggpi#1#2{{{E_\pi}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEgpi#1#2{{{E_\pi}\!\Bigl\{#1\Bigm|#2\Bigr\}}}

\def\hatCE#1#2{{\hat {E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEgg#1#2{{{E}\biggl\{#1\biggm|#2\biggr\}}}

\def\E#1{{{E}\left\{#1\right\}}}

\def\Epi#1{{{E_\pi}\!\left\{#1\right\}}}

\def\Eggpi#1{{{E_\pi}\!\biggl\{#1\biggr\}}}

\def\Epip#1{{{E_{\pi^\prime}}\!\left\{#1\right\}}}

\def\Ea#1{{{E_a}\!\left\{#1\right\}}}

\def\Egg#1{{{E}\biggl\{#1\biggr\}}}

\def\CP#1#2{{{Pr}\left\{{#1}{\mid}{#2}\right\}}}

\def\Pr#1{{{Pr}\left\{#1\right\}}}

\def\w{{\bf w}}

\def\cross{\times}

\def\grad{\nabla}

\def\first#1#2{{\partial #1\over \partial #2}}

\def\second#1#2{{\partial^2 #1\over \partial #2^2}}

\def\implies{\Rightarrow}

\def\union{\cup}

\def\bibliosection#1{%
  \addtocounter{bibliosection}{1}%
  \medskip\goodbreak
  {\bf \arabic{chapter}.\arabic{bibliosection} #1}
  \smallskip}

\def\skipbibliosection{\addtocounter{bibliosection}{1}}

\def\example{
  \addtocounter{example}{1}
  \smallskip
  \noindent
  {\bf Example \arabic{chapter}.\arabic{example}}~~}

\def\exercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}}~~ }

\def\pexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}~(programming)~~}}

\def\starredexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}*~}}

\def\lastexample{\arabic{chapter}.\arabic{example}}

\def\lastexercise{\arabic{chapter}.\arabic{exercise}}

\def\resetmycounters{
  \setcounter{bibliosection}{0}
  \setcounter{example}{0}
  \setcounter{exercise}{0}}

\def\myfootnote#1{\footnote{#1}}

\def\lambdatarget{7.3}

\def\polefigure{3.2}

\def\StateAggregation{8.4}

\def\banditchap{2}

\def\probchap{3}

\def\DPchap{4}

\def\MCchap{5}

\def\TDchap{6}

\def\ETchap{7}

\def\FAchap{8}

\def\modelchap{9}

\def\CSchap{11}

\def\Sumchap{10}

\def\finalchap{10}

\def\eqincN{\banditchap.4}

\def\eqAverage{\banditchap.1}

\def\eqSAconditions{\banditchap.7}

\def\secinc{\banditchap.5}

\def\secnonstationary{\banditchap.6}

\def\softmaxsection{\banditchap.3}

\def\RCsection{2.8}

\def\predictionbackups{\probchap.4}

\def\DPbackupVpi{\probchap.4a}

\def\undiscountedsection{\probchap.?}

\def\maxbackups{\probchap.7}

\def\BellmanVpi{\probchap.10}

\def\PImprov{\DPchap.2}

\def\GPIsec{\DPchap.6}

\def\EstimateOtherPolicySec{\MCchap.5}

\def\TDprediction{\TDchap.2}

\def\simpleTDrule{\TDchap.2}

\def\randomwalkexample{\TDchap.2}

\def\randomwalkfigure{\TDchap.4}

\def\sarsasection{\TDchap.3}

\def\ACsection{\TDchap.6}

\def\Rlearningsection{\TDchap.7}

\def\afterstatesection{\TDchap.8}

\def\ACrule{\TDchap.8}

\def\ACruleplus{\TDchap.9}

\def\TileCodingSection{\FAchap.3.2}

\def\sarsalalg{\FAchap.8}

\def\lambdacomparison{\FAchap.10}

\def\heuristicsearchsection{\modelchap.7}

\def\planspacesection{\modelchap.10}

\def\uniformvsonpolicysection{\modelchap.9}

\def\ZDcasestudy{\CSchap.6}

\def\samuelsection{\CSchap.2}

\def\TDGammonSection{\CSchap.1}

\def\g{\mbox{$\gamma$}}

\def\a{\mbox{$\alpha$}}

\def\l{\mbox{$\lambda$}}

\def\e{\mbox{$\epsilon$}}

\def\ee{\mbox{$\varepsilon$}}

\def\n{\mbox{$n$}}

\def\r{\mbox{$R$}}

\def\rl{R^{\lambda}}

\def\sp{{s^\prime}}

\def\ap{{a^\prime}}

\def\TDl{TD($\lambda$)}

\def\itTDl{{\it TD\/}\mbox{\rm ($\lambda$)}}

\def\Ql{Q($\lambda$)}

\def\itQl{{\it Q\/}\mbox{\rm ($\lambda$)}}

\def\sarsal{Sarsa($\lambda$)}

\def\itsarsal{{\it Sarsa\/}\mbox{\rm ($\lambda$)}}

\def\={\!=\!}

\def\X{{\cal X}}

\def\A{{\cal A}}

\def\S{{\cal S}}

\def\P{{\cal P}}

\def\R{{\cal R}}

\def\Pssa{\P_{s\sp}^a}

\def\Rssa{\R_{s\sp}^a}

\def\endofexample{$\diamondsuit$}

\def\starredsection#1{\section{#1\ (*)}}

\def\&{and}

\def\figbody#1#2#3#4#5{%








\centerline{\psfig{figure=#1.ps,rheight=#3,#5}}}

\def\comment#1{}

\def\myboxit#1{\vbox{\hrule\hbox{\vrule\kern6pt
             \vbox{\kern-17pt#1\kern-12pt}\kern8pt\vrule}\hrule}}

\makeatother
\newenvironment{tex2html_wrap}{}{}
\newwrite\lthtmlwrite
\def\lthtmltypeout#1{{\let\protect\string\immediate\write\lthtmlwrite{#1}}}%
\newbox\sizebox
\begin{document}
\pagestyle{empty}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Model(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1006: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\setcounter{bibliosection}{0}
\setcounter{example}{0}
\setcounter{exercise}{0}
\setcounter{chapter}{8}
\stepcounter{chapter}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1014: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$P^a_{s{s^\prime}
 }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1016: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$R^a_{s{s^\prime}
 }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1018: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \psfig{figure=model-planning-policy.ps,rheight=0.35in,width=2.224in}
}

{\newpage
\clearpage
\samepage \psfig{figure=common-structure.ps,rheight=0.35in,width=4.632in}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1020
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1020: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1022: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1024: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1028: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s,a,{s^\prime}
 ,r$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1032: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1034
$ Q(s,a) \gets Q(s,a) + 
                \alpha \left[r+\mbox{$\gamma$}
 \max_{a^\prime}
  Q({s^\prime}
 ,{a^\prime}
 )-Q(s,a)\right]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1034: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}
  
  \label{fig:QPlanning}
\end{figure}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=circle.ps,rheight=2.0in,width=2.916in}}

  \vspace*{-.1in}
  
  \label{fig:circle}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t,a_t
\leadsto s_{t+1},r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1040: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_t,a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1042: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s_{t+1},r_{t+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1044: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=dyna.ps,rheight=2.56in,width=3.55in}}

  
  \label{fig:dyna}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1048: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Model(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1050: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s\in\S$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1052: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a\in{\cal A}
 (s)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1054: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \mbox{current (non-terminal) state}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1056: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \epsilon\mbox{-greedy}(s,Q)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1058: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1062: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1066
$ Q(s,a) \gets Q(s,a) + 
              \alpha \left[r+\mbox{$\gamma$}
 \max_{a^\prime}
  Q({s^\prime}
 ,{a^\prime}
 )-Q(s,a)\right] $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1066: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Model(s,a) \gets {s^\prime}
 ,r$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1068: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \mbox{random previously observed state}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1072: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets \mbox{random action previously taken in $s$}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1074: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${s^\prime}
 ,r \gets Model(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1076: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1078
$Q(s,a) \gets Q(s,a) + 
                \alpha \left[r+\mbox{$\gamma$}
 \max_{a^\prime}
  Q({s^\prime}
 ,{a^\prime}
 )-Q(s,a)\right] $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1078: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Model(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1080: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Model(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1080: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}
  
  \label{fig:dyna-alg}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1128
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1129
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\gamma=.95$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1086: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=dyna-results.ps,rheight=3.16in,width=4.51in}}

  
  \label{fig:dyna-results}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1088
$\alpha=0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1088: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon=0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1090: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1102
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1102: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1104: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1106: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=dyna-snapshot.ps,rheight=1.512in,width=4.136in}}

  
  \label{fig:dyna-snapshot}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1118: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1130
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1131
\arabic{exercise}
}

\stepcounter{section}
\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1154
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1155
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1132: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=blocking.ps,rheight=2.9in,width=3.375in}}

  
  \label{fig:blocking}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1156
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1157
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1134: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1136: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=shortcut.ps,rheight=2.871in,width=3.5in}}

  
  \label{fig:shortcut}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r + \kappa \sqrt{n}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1142: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\kappa$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1144: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1158
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1159
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1160
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1161
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1162
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1163
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\kappa \sqrt{n}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1146: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a) + \kappa \sqrt{n_{sa}}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1148: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1164: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Model(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1166: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s \gets \mbox{current (nonterminal) state}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1172: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a \gets policy(s,Q)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1174: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1178: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Model(s,a) \gets {s^\prime}
 ,r$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1182: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p \gets |r+\mbox{$\gamma$}
 \max_{a^\prime}
  Q({s^\prime}
 ,{a^\prime}
 )-Q(s,a)|$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1184: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1186
$p>\theta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1186: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$s,a \gets first(PQueue)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1198: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${s^\prime}
 ,r \gets Model(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1200: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1202
$Q(s,a) \gets Q(s,a) + 
                \alpha \left[r+\mbox{$\gamma$}
 \max_{a^\prime}
  Q({s^\prime}
 ,{a^\prime}
 )-Q(s,a)\right] $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1202: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\bar s,\bar a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1204: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\bar r \gets \mbox{predicted reward}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1208: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p \gets |\bar r+\mbox{$\gamma$}
 \max_a Q(s,a)-Q(\bar s,\bar a)|$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1210: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1212
$p>\theta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1212: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\bar s,\bar a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1214: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\vbox{\hrule\hbox{\vrule\kern6pt
\vbox{\kern-17pt\box4\kern-12pt}\kern8pt\vrule}\hrule}}
  
  \label{fig:pri-sweep-alg}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1232
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1233
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1222: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=dyna-queue-results.ps,rheight=2.7truein,width=2.828in}}

  
  \label{fig:dyna-queue-results}
\end{figure}
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=rod.ps,rheight=2.75truein,width=2.75in}}

  
  \label{fig:rod}
\end{figure}
}

\addtocounter{example}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1234
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1235
\arabic{example}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$20 \times
20$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1224: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\diamondsuit$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1226: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}[p]

\centerline{\psfig{figure=full-and-sample.ps,rheight=6.64in,width=4.66125in}}


\label{fig:full-and-sample}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1236: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1238: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1240: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1242: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1244: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1246: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1248: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1250: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1252: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\hat P^a_{s{s^\prime}
 }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1256: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\hat
R^a_{s{s^\prime}
 }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1258: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:full-backup}
Q(s,a) \gets 
 \sum_{s^\prime}
  \hat P^a_{s{s^\prime}
 } \left[\hat R^a_{s{s^\prime}
 } + \gamma \max_{{a^\prime}
 } Q({s^\prime}
 ,{a^\prime}
 )
\right].
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1264: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 460
\label{eq:sample-backup}
Q(s,a) \gets (1-\alpha) Q(s,a) +
 \alpha \left[\hat R^a_{s{s^\prime}
 } + \gamma \max_{{a^\prime}
 } Q({s^\prime}
 ,{a^\prime}
 ) \right],
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1266
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1266: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\hat R^a_{s{s^\prime}
 }$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1268: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1270
$\alpha=1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1270: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q(s,a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1272: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q({s^\prime}
 ,{a^\prime}
 )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1274: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\sp$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1282: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\hat
P^a_{s{s^\prime}
 }>0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1284: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{${1\over\sqrt{t}}{b-1\over b}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1298: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1302
$\alpha=1/t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1302: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=sampling.ps,rheight=2.44in,width=4.545in}}


\label{fig:sampling}
\end{figure}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1318
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1319
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}[p]

\centerline{\psfig{figure=sampling2.ps,rheight=5.9in,width=3.752in}}


\label{fig:sampling2}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1322: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon=.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1324: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$|\S|$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1326: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^{\tilde\pi}(s_0)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1334: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tilde\pi$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1336: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1346
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1347
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1348
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1349
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$V^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1350: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1352: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1354: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\g^k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1358: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\lambda$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1360: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=backup-tree.ps,rheight=2.394in,width=4.4in}}


\label{fig:backup-tree}
\end{figure}
}

\stepcounter{section}
\stepcounter{section}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1364
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1365
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1366
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1367
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1368
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1369
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1370
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1371
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1372
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1373
\arabic{bibliosection}
}

\stepcounter{chapter}

\end{document}
