\batchmode
\documentstyle[12pt]{book}
\makeatletter


 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
%













\newcounter{bibliosection}

  


\newcounter{example}


\newcounter{exercise}





  





  






















































































\input psfig   %








\def\CE#1#2{{{E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpi#1#2{{{E_\pi}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpip#1#2{{{E_{\pi^\prime}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEpistar#1#2{{{E_{\pi^*}}\!\left\{{#1}{\mid}{#2}\right\}}}

\def\CEggpistar#1#2{{{E_{\pi^*}}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEggpi#1#2{{{E_\pi}\!\biggl\{#1\biggm|#2\biggr\}}}

\def\CEgpi#1#2{{{E_\pi}\!\Bigl\{#1\Bigm|#2\Bigr\}}}

\def\hatCE#1#2{{\hat {E}\left\{{#1}{\mid}{#2}\right\}}}

\def\CEgg#1#2{{{E}\biggl\{#1\biggm|#2\biggr\}}}

\def\E#1{{{E}\left\{#1\right\}}}

\def\Epi#1{{{E_\pi}\!\left\{#1\right\}}}

\def\Eggpi#1{{{E_\pi}\!\biggl\{#1\biggr\}}}

\def\Epip#1{{{E_{\pi^\prime}}\!\left\{#1\right\}}}

\def\Ea#1{{{E_a}\!\left\{#1\right\}}}

\def\Egg#1{{{E}\biggl\{#1\biggr\}}}

\def\CP#1#2{{{Pr}\left\{{#1}{\mid}{#2}\right\}}}

\def\Pr#1{{{Pr}\left\{#1\right\}}}

\def\w{{\bf w}}

\def\cross{\times}

\def\grad{\nabla}

\def\first#1#2{{\partial #1\over \partial #2}}

\def\second#1#2{{\partial^2 #1\over \partial #2^2}}

\def\implies{\Rightarrow}

\def\union{\cup}

\def\bibliosection#1{%
  \addtocounter{bibliosection}{1}%
  \medskip\goodbreak
  {\bf \arabic{chapter}.\arabic{bibliosection} #1}
  \smallskip}

\def\skipbibliosection{\addtocounter{bibliosection}{1}}

\def\example{
  \addtocounter{example}{1}
  \smallskip
  \noindent
  {\bf Example \arabic{chapter}.\arabic{example}}~~}

\def\exercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}}~~ }

\def\pexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}~(programming)~~}}

\def\starredexercise{
  \addtocounter{exercise}{1}
  \smallskip
  \noindent
  {\bf Exercise \arabic{chapter}.\arabic{exercise}*~}}

\def\lastexample{\arabic{chapter}.\arabic{example}}

\def\lastexercise{\arabic{chapter}.\arabic{exercise}}

\def\resetmycounters{
  \setcounter{bibliosection}{0}
  \setcounter{example}{0}
  \setcounter{exercise}{0}}

\def\myfootnote#1{\footnote{#1}}

\def\lambdatarget{7.3}

\def\polefigure{3.2}

\def\StateAggregation{8.4}

\def\banditchap{2}

\def\probchap{3}

\def\DPchap{4}

\def\MCchap{5}

\def\TDchap{6}

\def\ETchap{7}

\def\FAchap{8}

\def\modelchap{9}

\def\CSchap{11}

\def\Sumchap{10}

\def\finalchap{10}

\def\eqincN{\banditchap.4}

\def\eqAverage{\banditchap.1}

\def\eqSAconditions{\banditchap.7}

\def\secinc{\banditchap.5}

\def\secnonstationary{\banditchap.6}

\def\softmaxsection{\banditchap.3}

\def\RCsection{2.8}

\def\predictionbackups{\probchap.4}

\def\DPbackupVpi{\probchap.4a}

\def\undiscountedsection{\probchap.?}

\def\maxbackups{\probchap.7}

\def\BellmanVpi{\probchap.10}

\def\PImprov{\DPchap.2}

\def\GPIsec{\DPchap.6}

\def\EstimateOtherPolicySec{\MCchap.5}

\def\TDprediction{\TDchap.2}

\def\simpleTDrule{\TDchap.2}

\def\randomwalkexample{\TDchap.2}

\def\randomwalkfigure{\TDchap.4}

\def\sarsasection{\TDchap.3}

\def\ACsection{\TDchap.6}

\def\Rlearningsection{\TDchap.7}

\def\afterstatesection{\TDchap.8}

\def\ACrule{\TDchap.8}

\def\ACruleplus{\TDchap.9}

\def\TileCodingSection{\FAchap.3.2}

\def\sarsalalg{\FAchap.8}

\def\lambdacomparison{\FAchap.10}

\def\heuristicsearchsection{\modelchap.7}

\def\planspacesection{\modelchap.10}

\def\uniformvsonpolicysection{\modelchap.9}

\def\ZDcasestudy{\CSchap.6}

\def\samuelsection{\CSchap.2}

\def\TDGammonSection{\CSchap.1}

\def\g{\mbox{$\gamma$}}

\def\a{\mbox{$\alpha$}}

\def\l{\mbox{$\lambda$}}

\def\e{\mbox{$\epsilon$}}

\def\ee{\mbox{$\varepsilon$}}

\def\n{\mbox{$n$}}

\def\r{\mbox{$R$}}

\def\rl{R^{\lambda}}

\def\sp{{s^\prime}}

\def\ap{{a^\prime}}

\def\TDl{TD($\lambda$)}

\def\itTDl{{\it TD\/}\mbox{\rm ($\lambda$)}}

\def\Ql{Q($\lambda$)}

\def\itQl{{\it Q\/}\mbox{\rm ($\lambda$)}}

\def\sarsal{Sarsa($\lambda$)}

\def\itsarsal{{\it Sarsa\/}\mbox{\rm ($\lambda$)}}

\def\={\!=\!}

\def\X{{\cal X}}

\def\A{{\cal A}}

\def\S{{\cal S}}

\def\P{{\cal P}}

\def\R{{\cal R}}

\def\Pssa{\P_{s\sp}^a}

\def\Rssa{\R_{s\sp}^a}

\def\endofexample{$\diamondsuit$}

\def\starredsection#1{\section{#1\ (*)}}

\def\&{and}

\def\figbody#1#2#3#4#5{%








\centerline{\psfig{figure=#1.ps,rheight=#3,#5}}}

\def\comment#1{}

\def\Lri{\mbox{$L_{R-I}$}}

\def\Lrp{\mbox{$L_{R-P}$}}

\makeatother
\newenvironment{tex2html_wrap}{}{}
\newwrite\lthtmlwrite
\def\lthtmltypeout#1{{\let\protect\string\immediate\write\lthtmlwrite{#1}}}%
\newbox\sizebox
\begin{document}
\pagestyle{empty}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon $}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1094: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1096
$\alpha =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1096: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\setcounter{bibliosection}{0}
\setcounter{example}{0}
\setcounter{exercise}{0}
\setcounter{chapter}{1}
\stepcounter{chapter}
\stepcounter{section}
\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1122: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1126: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$k_a$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1132: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_1, r_2, \ldots, r_{k_a}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1134: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:sample-average}
  Q_t(a) = {r_1 + r_2 + \cdots + r_{k_a} \over k_a}.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$k_a =0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1136: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1138: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_0(a)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1140: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$k_a \rightarrow \infty$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1142: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1144: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1146: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1150: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_{t-1}(a_t^*) = \max_a Q_{t-1}(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1152: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1154: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1156: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$k_a\rightarrow\infty$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1158: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1162: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1164: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1-\e$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1166: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1168: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1176: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1182: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1188: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1188: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=bandits-graphs.ps,rheight=4.5in,width=4.641in}}


\label{fig:bandits-graphs}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1190: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =.01$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1192: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1194: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1196: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1198: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =.01$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1200: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1202: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1204: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1206: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1208: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1212
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1213
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1214: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}{ e^{Q_{t-1}(a)/\tau} \over \sum_b e^{Q_{t-1}(b)/\tau} },
  \label{eq:Gibbs}
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1220: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau\rightarrow 0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1222: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_{t-1}(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1224: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1226: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1228: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1230: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1238
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1239
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1234: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1240
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1241
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1236: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=contingency-space.ps,rheight=2.4in,width=3.186in}}


\label{fig:contingency-space}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1246: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1248: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=2bandits-graphs.ps,rheight=4.5in,width=4.627in}}


\label{fig:2bandits-graphs}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-I}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1250: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-P}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1252: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi_t(1)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1254: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi_t(2)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1256: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-P}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1258: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$d_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1262: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi_t(d_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1264: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1266
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1266: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 355
\label{eq:learning-automata}
  \pi_{t+1}(d_t) = \pi_t(d_t) + \alpha (1 - \pi_t(d_t)).
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1268
$\mbox{$\alpha$}
 =0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1268: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-P}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1270: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-P}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1272: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1276
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1276: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-P}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1278: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-I}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1280: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-P}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1282: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-P}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1284: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$L_{R-I}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1286: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1302
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1303
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \begin{displaymath}\hspace*{1in}
Q_t(a) = {r_1 + r_2 + \cdots + r_{k_a} \over k_a},
\hspace*{1.5in}\mbox{\rm (2.1)}
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_1,\ldots,r_{k_a}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1310: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1318: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1320: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(k+1)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1324: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_{k+1}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1326: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}\label{eq:incN}
 Q_{k+1} &=& {1\over k+1} \sum_{i=1}^{k+1} r_i\nonumber\\ 
         &=& {1\over k+1} \Bigl[r_{k+1} + \sum_{i=1}^{n} r_i \Bigr]\nonumber\\ 
         &=& {1\over k+1} \Bigl[r_{k+1} + k Q_k + Q_k - Q_k \Bigr]\nonumber\\ 
         &=& {1\over k+1} \Bigl[r_{k+1} + (k+1) Q_k - Q_k \Bigr]\nonumber\\ 
         &=& Q_k + {1\over k+1} [r_{k+1} - Q_k ],
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_1=r_1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1332: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1334: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1336: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{displaymath}\mbox{\sl New-estimate} \gets \mbox{\sl Old-estimate} ~+~ \mbox{\sl Step-size}~
[\mbox{\sl Target} - \mbox{\sl Old-estimate}].
\end{displaymath}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$[\mbox{\sl Target} - \mbox{\sl Old-estimate}]$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1340: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(k+1)\mbox{st}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1342: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$1\over k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1348: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1350
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1350: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\a_k(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1352: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\a_k(a)={1\over k_a}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1354: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1356
$\mbox{$\alpha$}
 =1/k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1356: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1366
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1367
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1360
$\mbox{$\alpha$}
 =1/k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1360: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$bandit(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1362: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1368: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 405
Q_{k+1} = Q_k + \alpha [r_{k+1} - Q_k]
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1372
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1372: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1374
$0<\alpha\le 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1374: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1376: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1378: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{eqnarray}% latex2html id marker 409
\label{eq:expave}
Q_k &=& Q_{k-1} + \alpha [r_k - Q_{k-1}] \nonumber\\ 
    &=& \alpha r_k + (1-\alpha) Q_{k-1} \nonumber\\ 
    &=& \alpha r_k + (1-\alpha)\alpha r_{k-1} + (1-\alpha)^2 Q_{k-2} \nonumber\\ 
    &=& \alpha r_k + (1-\alpha)\alpha r_{k-1} + (1-\alpha)^2\alpha r_{k-2} +
\nonumber\\ 
    & & ~~~~~~~~~~~~~~~\cdots + (1-\alpha)^{k-1}\alpha r_1 + (1-\alpha)^k Q_0
\nonumber\\ 
    &=& (1-\alpha)^k Q_0 + \sum_{i=1}^k \alpha(1-\alpha)^{k-i} r_i
\end{eqnarray}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1380
$(1-\alpha)^k +
\sum_{i=1}^k \alpha (1-\alpha)^{k-i} = 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1380: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1382
$\alpha(1-\alpha)^{k-i}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1382: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_i$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1384: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1388
$1-\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1388: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_i$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1392: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1394
$1-\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1394: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1396
$\alpha_k(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1396: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\a_k(a)=1/k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1402: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1404
$\{\alpha_k(a)\}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1404: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 426
\label{eq:convergence_conditions}
\sum_{k=1}^\infty \alpha_k(a) = \infty
\mbox{~~~~~~~~and~~~~~~~~}
\sum_{k=1}^\infty \alpha^2_k(a) < \infty.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1406
$\alpha_k(a)=1/k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1406: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1408
$\alpha_k(a)=\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1408: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1422
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1423
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1410
$\alpha_k(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1410: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_k(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1412: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1424
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1425
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1414: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1416
$\mbox{$\alpha$}
 =1/k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1416: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1418
$\mbox{$\alpha$}
 =0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1418: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1420: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_0(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1426: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1428
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1428: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q^*(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1430: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1432
$\alpha=.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1432: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1432
$\alpha=.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1432: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=initial-opt.ps,rheight=2.1in,width=4.64in}}


\label{fig:initial-opt}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_0(a)=+5$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1434: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1438: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_0(a)=0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1440: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1442
$\alpha=.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1442: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1448
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1449
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p_t(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1454: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 453
\label{eq:preferenceGibbs}
\pi_{t}(a)= {{Pr}\left\{a_{t}\=a\right\}}
 = {e^{p_{t-1}(a)} \over \sum_b e^{p_{t-1}(b)} },
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi_t(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1456: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1462: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1464: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\bar r_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1466: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:RC}
  p_{t+1}(a_t) = p_t(a_t) + \beta (r_t - \bar r_t),
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1468: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}% latex2html id marker 466
\label{eq:RCrbar}
  \bar r_{t+1} = \bar r_t + \alpha (r_t - \bar r_t),
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1470
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1470: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1472
$0<\alpha\le 1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1472: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\bar r_0$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1474: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1476
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1476: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=RC.ps,rheight=2.1in,width=4.64in}}


\label{fig:RC}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1478
$\alpha=.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1478: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1480: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1482: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1484
$\mbox{$\alpha$}
 =0.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1484: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1486
$\mbox{$\alpha$}
 =1/k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1486: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1504
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1505
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1488: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1490: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1506
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1507
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1492
$\alpha$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1492: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1494: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1496
$\mbox{$\alpha$}
 =\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1496: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1508
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1509
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\bar r$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1498: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(1-\pi_t(a_t))$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1500: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$p_t(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1512: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t^* =
\arg\max_a Q_{t-1}(a)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1518: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_{t}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1520: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t=a_t^*$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1522: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1524: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:pursuit1}
  \pi_{t}(a_t^*) = \pi_{t-1}(a_t^*) + \beta (1 - \pi_{t-1}(a_t^*) ),
\end{equation}
}

{\newpage
\clearpage
\samepage \begin{equation}\label{eq:pursuit2}
  \pi_{t}(a) = \pi_{t-1}(a) + \beta (0 - \pi_{t-1}(a) ), 
       \qquad \mbox{\rm for all~~} a\not=a_t^*.
\end{equation}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$a_t$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1526: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$Q_t(a_t)$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1528: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \begin{figure}\centerline{\psfig{figure=pursuit.ps,rheight=2.1in,width=4.64in}}


\label{fig:pursuit}
\end{figure}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{% latex2html id marker 1530
$\mbox{$\alpha$}
 =1/k$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1530: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\pi_0(a)=1/n$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1532: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\beta$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1536: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1538: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\mbox{$\epsilon$}
 =.1$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1540: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1550
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1551
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1542: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1544: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1552
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1553
\arabic{exercise}
}

\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1554
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1555
\arabic{exercise}
}

\addtocounter{exercise}{-1}
{\newpage
\clearpage
\samepage % latex2html id marker 1556
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1557
\arabic{exercise}
}

\addtocounter{exercise}{1}
\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1558
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1559
\arabic{exercise}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1546: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
\addtocounter{exercise}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1572
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1573
\arabic{exercise}
}

\stepcounter{section}
{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1574: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$(1-\mbox{$\epsilon$}
 )$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1576: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$2^{2000}$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1578: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\stepcounter{section}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1588
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1589
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1590
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1591
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\epsilon$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1584: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1592
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1593
\arabic{bibliosection}
}

{\newpage
\clearpage
\samepage \setbox\sizebox=\hbox{$\tau$}\lthtmltypeout{latex2htmlSize :tex2html_wrap_inline1586: \the\ht\sizebox::\the\dp\sizebox.}\box\sizebox
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1594
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1595
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1596
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1597
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1598
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1599
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1600
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1601
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1602
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1603
\arabic{bibliosection}
}

\addtocounter{bibliosection}{1}
{\newpage
\clearpage
\samepage % latex2html id marker 1604
\arabic{chapter}
}

{\newpage
\clearpage
\samepage % latex2html id marker 1605
\arabic{bibliosection}
}

\stepcounter{chapter}

\end{document}
