\begin{thebibliography}{}

\bibitem[Agre, 1988]{Agre-thesis}
Agre, P.~E. (1988).
\newblock {\em The Dynamic Structure of Everyday Life}.
\newblock PhD thesis, Massachusetts Institute of Technology, Cambridge, MA.
\newblock AI-TR 1085, MIT Artificial Intelligence Laboratory.

\bibitem[Agre and Chapman, 1990]{Agre-Chapman}
Agre, P.~E. and Chapman, D. (1990).
\newblock What are plans for?
\newblock {\em Robotics and Autonomous Systems}, 6:17--34.

\bibitem[Albus, 1971]{Albus-cerebellum}
Albus, J.~S. (1971).
\newblock A theory of cerebellar function.
\newblock {\em Mathematical Biosciences}, 10:25--61.

\bibitem[Albus, 1981]{Albus-book}
Albus, J.~S. (1981).
\newblock {\em Brain, Behavior, and Robotics}.
\newblock Byte Books.

\bibitem[Anderson, 1986]{Anderson-thesis}
Anderson, C.~W. (1986).
\newblock {\em Learning and Problem Solving with Multilayer Connectionist
  Systems}.
\newblock PhD thesis, University of Massachusetts, Amherst, MA.

\bibitem[Anderson, 1987]{Anderson-ML}
Anderson, C.~W. (1987).
\newblock Strategy learning with multilayer connectionist representations.
\newblock Technical Report TR87-509.3, GTE Laboratories, Incorporated, Waltham,
  MA.
\newblock (This is a corrected version of the report published in {\it
  Proceedings of the Fourth International Workshop on Machine
  Learning},103--114, 1987, San Mateo, CA: Morgan Kaufmann.).

\bibitem[Anderson et~al., 1977]{Anderson-Silversten-Ritz-Jones}
Anderson, J.~A., Silversten, J.~W., Ritz, S.~A., and Jones, R.~S. (1977).
\newblock Distinctive features, categorical perception, and probability
  learning: {S}ome applications of a neural model.
\newblock {\em Psychological Review}, 84:413--451.

\bibitem[Andreae, 1963]{Andreae-STeLLA}
Andreae, J.~H. (1963).
\newblock {STELLA}: {A} scheme for a learning machine.
\newblock In {\em Proceedings of the 2nd IFAC Congress, Basle}, pages 497--502,
  London. Butterworths.

\bibitem[Andreae, 1969a]{Andreae-Monologue}
Andreae, J.~H. (1969a).
\newblock A learning machine with monologue.
\newblock {\em International Journal of Man-Machine Studies}, 1:1--20.

\bibitem[Andreae, 1969b]{Andreae-encyclopedia}
Andreae, J.~H. (1969b).
\newblock Learning machines---a unified view.
\newblock In Meetham, A.~R. and Hudson, R.~A., editors, {\em Encyclopedia of
  Information, Linguistics, and Control}, pages 261--270. Pergamon, Oxford.

\bibitem[Andreae, 1977]{Andreae-book}
Andreae, J.~H. (1977).
\newblock {\em Thinking with the Teachable Machine}.
\newblock Academic Press, London.

\bibitem[Baird, 1995]{Baird-ml95}
Baird, L.~C. (1995).
\newblock Residual algorithms: {R}einforcement learning with function
  approximation.
\newblock In Prieditis, A. and Russell, S., editors, {\em Proceedings of the
  Twelfth International Conference on Machine Learning}, pages 30--37, San
  Francisco, CA. Morgan Kaufmann.

\bibitem[Bao et~al., 1994]{Bao-etal}
Bao, G., Cassandras, C.~G., Djaferis, T.~E., Gandhi, A.~D., and Looze, D.~P.
  (1994).
\newblock Elevator dispatchers for down peak traffic.
\newblock Technical report, ECE Department, University of Massachusetts.

\bibitem[Barnard, 1993]{Barnard-93}
Barnard, E. (1993).
\newblock Temporal-difference methods and {M}arkov models.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  23:357--365.

\bibitem[Barto, 1985]{HN}
Barto, A.~G. (1985).
\newblock Learning by statistical cooperation of self-interested neuron-like
  computing elements.
\newblock {\em Human Neurobiology}, 4:229--256.

\bibitem[Barto, 1986]{Barto-Snowbird-86}
Barto, A.~G. (1986).
\newblock Game-theoretic cooperativity in networks of self-interested units.
\newblock In Denker, J.~S., editor, {\em Neural Networks for Computing}, pages
  41--46. American Institute of Physics, New York.

\bibitem[Barto, 1990]{Barto-control}
Barto, A.~G. (1990).
\newblock Connectionist learning for control: {A}n overview.
\newblock In Miller, T., Sutton, R.~S., and Werbos, P.~J., editors, {\em Neural
  Networks for Control}, pages 5--58. MIT Press, Cambridge, MA.

\bibitem[Barto, 1991]{Barto-cs}
Barto, A.~G. (1991).
\newblock Some learning tasks from a control perspective.
\newblock In Nadel, L. and Stein, D.~L., editors, {\em 1990 Lectures in Complex
  Systems}, pages 195--223. Addison-Wesley Publishing Company, The Advanced
  Book Program, Redwood City, CA.

\bibitem[Barto, 1992]{Barto-handbook}
Barto, A.~G. (1992).
\newblock Reinforcement learning and adaptive critic methods.
\newblock In White, D.~A. and Sofge, D.~A., editors, {\em Handbook of
  Intelligent Control: {N}eural, Fuzzy, and Adaptive Approaches}, pages
  469--491. Van Nostrand Reinhold, New York.

\bibitem[Barto, 1995a]{Barto-basalganglia}
Barto, A.~G. (1995a).
\newblock Adaptive critics and the basal ganglia.
\newblock In Houk, J.~C., Davis, J.~L., and Beiser, D.~G., editors, {\em Models
  of Information Processing in the Basal Ganglia}, pages 215--232. MIT Press,
  Cambridge, MA.

\bibitem[Barto, 1995b]{Barto-Arbib-handbook}
Barto, A.~G. (1995b).
\newblock Reinforcement learning.
\newblock In Arbib, M.~A., editor, {\em Handbook of Brain Theory and Neural
  Networks}, pages 804--809. The MIT Press, Cambridge, MA.

\bibitem[Barto and Anandan, 1985]{arp}
Barto, A.~G. and Anandan, P. (1985).
\newblock Pattern recognizing stochastic learning automata.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  15:360--375.

\bibitem[Barto and Anderson, 1985]{structural}
Barto, A.~G. and Anderson, C.~W. (1985).
\newblock Structural learning in connectionist systems.
\newblock In {\em Program of the Seventh Annual Conference of the Cognitive
  Science Society}, pages 43--54, Irvine, CA.

\bibitem[Barto et~al., 1982]{synthesis}
Barto, A.~G., Anderson, C.~W., and Sutton, R.~S. (1982).
\newblock Synthesis of nonlinear control surfaces by a layered associative
  search network.
\newblock {\em Biological Cybernetics}, 43:175--185.

\bibitem[Barto et~al., 1991]{Barto-Bradtke-Singh-TR}
Barto, A.~G., Bradtke, S.~J., and Singh, S.~P. (1991).
\newblock Real-time learning and control using asynchronous dynamic
  programming.
\newblock Technical Report 91-57, Department of Computer and Information
  Science, University of Massachusetts, Amherst, MA.

\bibitem[Barto et~al., 1995]{Barto-Bradtke-Singh}
Barto, A.~G., Bradtke, S.~J., and Singh, S.~P. (1995).
\newblock Learning to act using real-time dynamic programming.
\newblock {\em Artificial Intelligence}, 72:81--138.

\bibitem[Barto and Duff, 1994]{Barto-Duff-NIPS6}
Barto, A.~G. and Duff, M. (1994).
\newblock Monte carlo matrix inversion and reinforcement learning.
\newblock In Cohen, J.~D., Tesauro, G., and Alspector, J., editors, {\em
  Advances in Neural Information Processing Systems: {P}roceedings of the 1993
  Conference}, pages 687--694, San Francisco, CA. Morgan Kaufmann.

\bibitem[Barto and Jordan, 1987]{Barto-Jordan-ICNN87}
Barto, A.~G. and Jordan, M.~I. (1987).
\newblock Gradient following without back-propagation in layered networks.
\newblock In Caudill, M. and Butler, C., editors, {\em Proceedings of the
  {IEEE} First Annual Conference on Neural Networks}, pages II629--II636, San
  Diego, CA.

\bibitem[Barto and Sutton, 1981a]{final-report-81}
Barto, A.~G. and Sutton, R.~S. (1981a).
\newblock Goal seeking components for adaptive intelligence: {A}n initial
  assessment.
\newblock Technical Report AFWAL-TR-81-1070, Air Force Wright Aeronautical
  Laboratories/Avionics Laboratory, Wright-Patterson AFB, OH.

\bibitem[Barto and Sutton, 1981b]{landmark-learning}
Barto, A.~G. and Sutton, R.~S. (1981b).
\newblock Landmark learning: {A}n illustration of associative search.
\newblock {\em Biological Cybernetics}, 42:1--8.

\bibitem[Barto and Sutton, 1982]{Barto-Sutton-82}
Barto, A.~G. and Sutton, R.~S. (1982).
\newblock Simulation of anticipatory responses in classical conditioning by a
  neuron-like adaptive element.
\newblock {\em Behavioural Brain Research}, 4:221--235.

\bibitem[Barto et~al., 1983]{pole}
Barto, A.~G., Sutton, R.~S., and Anderson, C.~W. (1983).
\newblock Neuronlike elements that can solve difficult learning control
  problems.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  13:835--846.
\newblock Reprinted in J. A. Anderson and E. Rosenfeld, {\it Neurocomputing:
  Foundations of Research}, MIT Press, Cambridge, MA, 1988.

\bibitem[Barto et~al., 1981]{ASN}
Barto, A.~G., Sutton, R.~S., and Brouwer, P.~S. (1981).
\newblock Associative search network: {A} reinforcement learning associative
  memory.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  40:201--211.

\bibitem[Bellman and Dreyfus, 1959]{Bellman-Dreyfus}
Bellman, R. and Dreyfus, S.~E. (1959).
\newblock Functional approximations and dynamic programming.
\newblock {\em Math Tables and Other Aides to Computation}, 13:247--251.

\bibitem[Bellman et~al., 1973]{Bellman-Kalaba-Kotkin}
Bellman, R., Kalaba, R., and Kotkin, B. (1973).
\newblock Polynomial approximation---{A} new computational technique in dynamic
  programming: {A}llocation processes.
\newblock {\em Mathematical Computation}, 17:155--161.

\bibitem[Bellman, 1956]{Bellman-56}
Bellman, R.~E. (1956).
\newblock A problem in the sequential design of experiments.
\newblock {\em Sankhya}, 16:221--229.

\bibitem[Bellman, 1957a]{Bellman-book}
Bellman, R.~E. (1957a).
\newblock {\em Dynamic Programming}.
\newblock Princeton University Press, Princeton, NJ.

\bibitem[Bellman, 1957b]{Bellman-MDP}
Bellman, R.~E. (1957b).
\newblock A {M}arkov decision process.
\newblock {\em Journal of Mathematical Mech.}, 6:679--684.

\bibitem[Berry and Fristedt, 1985]{bandit}
Berry, D.~A. and Fristedt, B. (1985).
\newblock {\em Bandit Problems}.
\newblock Chapman and Hall, London.

\bibitem[Bertsekas, 1982]{Bertsekas-82}
Bertsekas, D.~P. (1982).
\newblock Distributed dynamic programming.
\newblock {\em IEEE Transactions on Automatic Control}, 27:610--616.

\bibitem[Bertsekas, 1983]{Bertsekas-83}
Bertsekas, D.~P. (1983).
\newblock Distributed asynchronous computation of fixed points.
\newblock {\em Mathematical Programming}, 27:107--120.

\bibitem[Bertsekas, 1987]{Bertsekas-book87}
Bertsekas, D.~P. (1987).
\newblock {\em Dynamic Programming: {D}eterministic and Stochastic Models}.
\newblock Prentice-Hall, Englewood Cliffs, NJ.

\bibitem[Bertsekas, 1995]{Bertsekas-book95}
Bertsekas, D.~P. (1995).
\newblock {\em Dynamic Programming and Optimal Control.}
\newblock Athena, Belmont, MA.

\bibitem[Bertsekas and Tsitsiklis, 1989]{Bertsekas-Tsitsiklis}
Bertsekas, D.~P. and Tsitsiklis, J.~N. (1989).
\newblock {\em Parallel and Distributed Computation: {N}umerical Methods}.
\newblock Prentice-Hall, Englewood Cliffs, NJ.

\bibitem[Bertsekas and Tsitsiklis, 1996]{NDP}
Bertsekas, D.~P. and Tsitsiklis, J.~N. (1996).
\newblock {\em Neural Dynamic Programming}.
\newblock Athena Scientific, Belmont, MA.

\bibitem[Biermann et~al., 1982]{Biermann-etal}
Biermann, A.~W., Fairfield, J. R.~C., and Beres, T.~R. (1982).
\newblock Signature table systems and learning.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  SMC-12:635--648.

\bibitem[Bishop, 1995]{Bishop}
Bishop, C.~M. (1995).
\newblock {\em Neural Networks for Pattern Recognition}.
\newblock Clarendon, Oxford.

\bibitem[Booker, 1982]{Booker}
Booker, L.~B. (1982).
\newblock {\em Intelligent Behavior as an Adaptation to the Task Environment}.
\newblock PhD thesis, University of Michigan, Ann Arbor, MI.

\bibitem[Boone, 1997]{Boone-acrobot}
Boone, G. (1997).
\newblock Minimum-time control of the acrobot.
\newblock In {\em 1997 International Conference on Robotics and Automation},
  Albuquerque, NM.

\bibitem[Boutilier et~al., 1995]{Boutilier-Dearden-Goldszmidt}
Boutilier, C., Dearden, R., and Goldszmidt, M. (1995).
\newblock Exploiting structure in policy construction.
\newblock In {\em Proceedings of the Fourteenth International Joint Conference
  on Artificial Intelligence}.

\bibitem[Boyan and Moore, 1995]{Boyan-Moore-NIPS7}
Boyan, J.~A. and Moore, A.~W. (1995).
\newblock Generalization in reinforcement learning: {S}afely approximating the
  value functions.
\newblock In G.~Tesauro, D.~Touretzky, T.~L., editor, {\em Advances in Neural
  Information Processing Systems: {P}roceedings of the 1994 Conference}, pages
  369--376, San Mateo, CA. Morgan Kaufmann.

\bibitem[Boyan et~al., 1995]{Boyan-etal-VFA}
Boyan, J.~A., Moore, A.~W., and Sutton, R.~S., editors (1995).
\newblock {\em Proceedings of the Workshop on Value Function Approximation.
  Machine Learning Conference 1995}, Pittsburgh, PA. School of Computer
  Science, Carnegie Mellon University.
\newblock Technical Report CMU-CS-95-206.

\bibitem[Bradtke, 1993]{Bradtke-NIPS5}
Bradtke, S.~J. (1993).
\newblock Reinforcement learning applied to linear quadratic regulation.
\newblock In S.~J.~Hanson, J. D.~Cowan, C. L.~G., editor, {\em Advances in
  Neural Information Processing Systems: {P}roceedings of the 1992 Conference},
  pages 295--302, San Mateo, CA. Morgan Kaufmann.

\bibitem[Bradtke, 1994]{Bradtke-thesis}
Bradtke, S.~J. (1994).
\newblock {\em Incremental Dynamic Programming for On-Line Adaptive Optimal
  Control}.
\newblock PhD thesis, University of Massachusetts, Amherst.
\newblock Appeared as CMPSCI Technical Report 94-62.

\bibitem[Bradtke and Barto, 1996]{Bradtke-Barto}
Bradtke, S.~J. and Barto, A.~G. (1996).
\newblock Linear least--squares algorithms for temporal difference learning.
\newblock {\em Machine Learning}, 22:33--57.

\bibitem[Bradtke and Duff, 1995]{Bradtke-Duff-NIPS7}
Bradtke, S.~J. and Duff, M.~O. (1995).
\newblock Reinforcement learning methods for continuous-time {M}arkov decision
  problems.
\newblock In G.~Tesauro, D.~Touretzky, T.~L., editor, {\em Advances in Neural
  Information Processing Systems: {P}roceedings of the 1994 Conference}, pages
  393--400, San Mateo, CA. Morgan Kaufmann.

\bibitem[Bridle, 1990]{Bridle}
Bridle, J.~S. (1990).
\newblock Training stochastic model recognition algorithms as networks can lead
  to maximum mutual information estimates of parameters.
\newblock In Touretzky, D.~S., editor, {\em Advances in Neural Information
  Processing Systems 2}, pages 211--217, San Mateo, CA. Morgan Kaufmann.

\bibitem[Broomhead and Lowe, 1988]{Broomhead-Lowe}
Broomhead, D.~S. and Lowe, D. (1988).
\newblock Multivariable functional interpolation and adaptive networks.
\newblock {\em Complex Systems}, 2:321--355.

\bibitem[Bryson, 1996]{Bryson-history}
Bryson, Jr., A.~E. (1996).
\newblock Optimal control---1950 to 1985.
\newblock {\em {IEEE} Control Systems}, 13(3):26--33.

\bibitem[Bush and Mosteller, 1955]{Bush-Mosteller}
Bush, R.~R. and Mosteller, F. (1955).
\newblock {\em Stochastic Models for Learning}.
\newblock Wiley, New York.

\bibitem[Byrne et~al., 1990]{Byrne-etal}
Byrne, J.~H., Gingrich, K.~J., and Baxter, D.~A. (1990).
\newblock Computational capabilities of single neurons: {R}elationship to
  simple forms of associative and nonassociative learning in {\it aplysia}.
\newblock In Hawkins, R.~D. and Bower, G.~H., editors, {\em Computational
  Models of Learning}, pages 31--63. Academic Press, New York.

\bibitem[Campbell, 1959]{Campbell}
Campbell, D.~T. (1959).
\newblock Blind variation and selective survival as a general strategy in
  knowledge-processes.
\newblock In Yovits, M.~C. and Cameron, S., editors, {\em Self-Organizing
  Systems}, pages 205--231. Pergamon.

\bibitem[{Carlstr\"om} and {Nordstr\"om}, 1997]{Carlstrom-Nordstrom}
{Carlstr\"om}, J. and {Nordstr\"om}, E. (1997).
\newblock Control of self-similar atm call traffic by reinforcement learning.
\newblock In {\em Proceedings of the International Workshop on Applications of
  Neural Networks to Telecommunications 3 (IWANNT*97)}, Hillsdale NJ. Lawrence
  Erlbaum.

\bibitem[Chapman and Kaelbling, 1991]{Chapman-Kaelbling-IJCAI91}
Chapman, D. and Kaelbling, L.~P. (1991).
\newblock Input generalization in delayed reinforcement learning: {A}n
  algorithm and performance comparisons.
\newblock In {\em Proceedings of the 1991 International Joint Conference on
  Artificial Intelligence}.

\bibitem[Chow and Tsitsiklis, 1991]{Chow-Tsitsiklis}
Chow, C.-S. and Tsitsiklis, J.~N. (1991).
\newblock An optimal one-way multigrid algorithm for discrete-time stochastic
  control.
\newblock {\em IEEE Transactions on Automatic Control}, 36:898--914.

\bibitem[Chrisman, 1992]{Chrisman}
Chrisman, L. (1992).
\newblock Reinforcement learning with perceptual aliasing: {T}he perceptual
  distinctions approach.
\newblock In {\em Proceedings of the Tenth National Conference on Artificial
  Intelligence}, pages 183--188, Menlo Park, CA. AAAI Press/MIT Press.

\bibitem[Christensen and Korf, 1986]{Christensen-Korf}
Christensen, J. and Korf, R.~E. (1986).
\newblock A unified theory of heuristic evaluation functions and its
  application to learning.
\newblock In {\em Proceedings of the Fifth National Conference on Artificial
  Intelligence {AAAI-86}}, pages 148--152, San Mateo, CA. Morgan Kaufmann.

\bibitem[Cichosz, 1995]{Cichosz-TTD}
Cichosz, P. (1995).
\newblock Truncating temporal differences: {O}n the efficient implementation of
  {TD}(lambda) for reinforcement learning.
\newblock {\em Journal of Artificial Intelligence Research}, 2:287--318.

\bibitem[Clark and Farley, 1955]{Clark-Farley55}
Clark, W.~A. and Farley, B.~G. (1955).
\newblock Generalization of pattern recognition in a self-organizing system.
\newblock In {\em Proceedings of the 1955 Western Joint Computer Conference},
  pages 86--91.

\bibitem[Clouse, 1997]{Clouse-thesis}
Clouse, J. (1997).
\newblock {\em On Integrating Apprentice Learning and Reinforcement Learning
  TITLE2}.
\newblock PhD thesis, University of Massachusetts, Amherst.
\newblock Appeared as CMPSCI Technical Report 96-026.

\bibitem[Clouse and Utgoff, 1992]{Clouse-Utgoff}
Clouse, J. and Utgoff, P. (1992).
\newblock A teaching method for reinforcement learning systems.
\newblock In {\em Proceedings of the Ninth International Machine Learning
  Conference}, pages 92--101.

\bibitem[Colombetti and Dorigo, 1994]{Colombetti-Dorigo}
Colombetti, M. and Dorigo, M. (1994).
\newblock Training agent to perform sequential behavior.
\newblock {\em Adaptive Behavior}, 2(3):247--275.

\bibitem[Connell, 1989]{Connell-can}
Connell, J. (1989).
\newblock A colony architecture for an artificial creature.
\newblock Technical Report Technical Report AI-TR-1151, MIT Artificial
  Intelligence Laboratory, Cambridge, MA.

\bibitem[Craik, 1943]{Craik}
Craik, K. J.~W. (1943).
\newblock {\em The Nature of Explanation}.
\newblock Cambridge University Press, Cambridge.

\bibitem[Crites, 1996]{Crites-thesis}
Crites, R.~H. (1996).
\newblock {\em Large-Scale Dynamic Optimization Using Teams of Reinforcement
  Learning Agents}.
\newblock PhD thesis, University of Massachusetts, Amherst, MA.

\bibitem[Crites and Barto, 1996]{Crites-Barto-NIPS8}
Crites, R.~H. and Barto, A.~G. (1996).
\newblock Improving elevator performance using reinforcement learning.
\newblock In D.~S.~Touretzky, M. C.~Mozer, M. E.~H., editor, {\em Advances in
  Neural Information Processing Systems: {P}roceedings of the 1995 Conference},
  pages 1017--1023, Cambridge, MA. MIT Press.

\bibitem[Curtiss, 1954]{Curtiss}
Curtiss, J.~H. (1954).
\newblock A theoretical comparison of the efficiencies of two classical methods
  and a monte carlo method for computing one component of the solution of a set
  of linear algebraic equations.
\newblock In Meyer, H.~A., editor, {\em Symposium on Monte Carlo Methods},
  pages 191--233. Wiley, New York.

\bibitem[Cziko, 1995]{Cziko}
Cziko, G. (1995).
\newblock {\em Without Miracles. Universal Selection Theory and the Second
  Darvinian Revolution}.
\newblock The MIT Press.

\bibitem[Daniel, 1976]{Daniel}
Daniel, J.~W. (1976).
\newblock Splines and efficiency in dynamic programming.
\newblock {\em Journal of Mathematical Analysis and Applications}, 54:402--407.

\bibitem[Dayan, 1991]{Dayan-91}
Dayan, P. (1991).
\newblock Reinforcement comparison.
\newblock In Touretzky, D.~S., Elman, J.~L., Sejnowski, T.~J., and Hinton,
  G.~E., editors, {\em Connectionist Models: {P}roceedings of the 1990 Summer
  School}, pages 45--51. Morgan Kaufmann, San Mateo, CA.

\bibitem[Dayan, 1992]{Dayan-ML}
Dayan, P. (1992).
\newblock The convergence of {TD}($\lambda$) for general $\lambda$.
\newblock {\em Machine Learning}, 8:341--362.

\bibitem[Dayan and Hinton, 1993]{Dayan-Hinton-NIPS5}
Dayan, P. and Hinton, G.~E. (1993).
\newblock Feudal reinforcement learning.
\newblock In Hanson, S.~J., Cohen, J.~D., and Giles, C.~L., editors, {\em
  Advances in Neural Information Processing Systems: {P}roceedings of the 1992
  Conference}, pages 271--278, San Mateo, CA. Morgan Kaufmann.

\bibitem[Dayan and Sejnowski, 1994]{Dayan-Sejnowski}
Dayan, P. and Sejnowski, T. (1994).
\newblock {TD}($\lambda$) converges with probability 1.
\newblock {\em Machine Learning}, 14:295--301.

\bibitem[Dean and Lin, 1995]{Dean-Lin}
Dean, T. and Lin, S.-H. (1995).
\newblock Decomposition techniques for planning in stochastic domains.
\newblock In {\em Proceedings of the Fourteenth International Joint Conference
  on Artificial Intelligence}.

\bibitem[DeJong and Spong, 1994]{DeJong-Spong}
DeJong, G. and Spong, M.~W. (1994).
\newblock Swinging up the acrobot: {A}n example of intelligent control.
\newblock In {\em Proceedings of the American Control Conference}, pages
  2158--2162.

\bibitem[Denardo, 1967]{Denardo}
Denardo, E.~V. (1967).
\newblock Contraction mappings in the theory underlying dynamic programming.
\newblock {\em {SIAM} Review}, 9:165--177.

\bibitem[Dennett, 1978]{Dennett-Law-of-Effect}
Dennett, D.~C. (1978).
\newblock {\em Brainstorms}, chapter Why the Law-of-Effect Will Not Go Away,
  pages 71--89.
\newblock Bradford/MIT Press, Cambridge, MA.

\bibitem[Dietterich and Flann, 1995]{Dietterich-Flann-ml95}
Dietterich, T.~G. and Flann, N.~S. (1995).
\newblock Explanation-based learning and reinforcement learning: {A} unified
  view.
\newblock In Prieditis, A. and Russell, S., editors, {\em Proceedings of the
  Twelfth International Conference on Machine Learning}, pages 176--184, San
  Francisco, CA. Morgan Kaufmann.

\bibitem[Doya, 1996]{Doya-NIPS8}
Doya, K. (1996).
\newblock Temporal difference learing in continuous time and space.
\newblock In Touretzky, D.~S., Mozer, M.~C., and Hasselmo, M.~E., editors, {\em
  Advances in Neural Information Processing Systems: {P}roceedings of the 1995
  Conference}, pages 1073--1079, Cambridge, MA. MIT Press.

\bibitem[Doyle and Snell, 1984]{Doyle-Snell}
Doyle, P.~G. and Snell, J.~L. (1984).
\newblock {\em Random Walks and Electric Networks}.
\newblock The Mathematical Association of America.
\newblock Carus Mathematical Monograph 22.

\bibitem[Dreyfus and Law, 1977]{Dreyfus-Law}
Dreyfus, S.~E. and Law, A.~M. (1977).
\newblock {\em The Art and Theory of Dynamic Programming}.
\newblock Academic Press, New York.

\bibitem[Duda and Hart, 1973]{Duda-Hart}
Duda, R.~O. and Hart, P.~E. (1973).
\newblock {\em Pattern Classification and Scene Analysis}.
\newblock Wiley, New York.

\bibitem[Duff, 1995]{Duff-bandits}
Duff, M.~O. (1995).
\newblock Q-learning for bandit problems.
\newblock In Prieditis, A. and Russell, S., editors, {\em Proceedings of the
  Twelfth International Conference on Machine Learning}, pages 209--217, San
  Francisco, CA. Morgan Kaufmann.

\bibitem[Estes, 1950]{Estes-50}
Estes, W.~K. (1950).
\newblock Toward a statistical theory of learning.
\newblock {\em Psychololgical Review}, 57:94--107.

\bibitem[Farley and Clark, 1954]{Farley-Clark}
Farley, B.~G. and Clark, W.~A. (1954).
\newblock Simulation of self-organizing systems by digital computer.
\newblock {\em {IRE} Transactions on Information Theory}, 4:76--84.

\bibitem[Feldbaum, 1960]{Feldbaum-65}
Feldbaum, A.~A. (1960).
\newblock {\em Optimal Control Theory}.
\newblock Academic Press, New York.

\bibitem[Friston et~al., 1994]{Friston-etal}
Friston, K.~J., Tononi, G., Reeke, G.~N., Sporns, O., and Edelman, G.~M.
  (1994).
\newblock Value-dependent selection in the brain: {S}imulation in a synthetic
  neural model.
\newblock {\em Neuroscience}, 59:229--243.

\bibitem[Fu, 1970]{Fu-70}
Fu, K.~S. (1970).
\newblock Learning control systems---{R}eview and outlook.
\newblock {\em IEEE Transactions on Automatic Control}, pages 210--221.

\bibitem[Galanter and Gerstenhaber, 1956]{Galanter-Gerstenhaber}
Galanter, E. and Gerstenhaber, M. (1956).
\newblock On thought: {T}he extrinsic theory.
\newblock {\em Psychological Review}, 63:218--227.

\bibitem[{G\"allmo} and Asplund, 1995]{Gallmo-Asplund}
{G\"allmo}, O. and Asplund, H. (1995).
\newblock Reinforcement learning by construction of hypothetical targets.
\newblock In Alspector, J., Goodman, R., and Brown, T.~X., editors, {\em
  Proceedings of the International Workshop on Applications of Neural Networks
  to Telecommunications 2 (IWANNT-2)}, pages 300--307. Stockholm, Sweden.

\bibitem[Gardner, 1981]{Gardner}
Gardner (1981).
\newblock Samuel's checkers player.
\newblock In Barr, A. and Feigenbaum, E.~A., editors, {\em The Handbook of
  Artificial Intelligence, {I}}, pages 84--108. William Kaufmann, Los Altos,
  CA.

\bibitem[Gardner, 1973]{race-track}
Gardner, M. (1973).
\newblock Mathematical games.
\newblock {\em Scientific American}, 228:108.

\bibitem[Gelperin et~al., 1985]{Gelperin-Hopfield-Tank}
Gelperin, A., Hopfield, J.~J., and Tank, D.~W. (1985).
\newblock The logic of {\em limax\/} learning.
\newblock In Selverston, A., editor, {\em Model Neural Networks and Behavior}.
  Plenum Press, New York.

\bibitem[Gittins and Jones, 1974]{Gittins-Jones-74}
Gittins, J.~C. and Jones, D.~M. (1974).
\newblock A dynamic allocation index for the sequential design of experiments.
\newblock {\em Progress in Statistics}, pages 241--266.

\bibitem[Goldberg, 1989]{Goldberg-book}
Goldberg, D.~E. (1989).
\newblock {\em Genetic Algorithms in Search, Optimization, and Machine
  Learning}.
\newblock Addison-Wesley, Reading, MA.

\bibitem[Goldstein, 1957]{Goldstein}
Goldstein, H. (1957).
\newblock {\em Classical Mechanics}.
\newblock Addison-Wesley, Rreadin, MA.

\bibitem[Goodwin and Sin, 1984]{Goodwin-Sin}
Goodwin, G.~C. and Sin, K.~S. (1984).
\newblock {\em Adaptive Filtering Prediction and Control}.
\newblock Prentice-Hall, Englewood Cliffs, N.J.

\bibitem[Gordon, 1995]{Gordon-ml95}
Gordon, G.~J. (1995).
\newblock Stable function approximation in dynamic programming.
\newblock In Prieditis, A. and Russell, S., editors, {\em Proceedings of the
  Twelfth International Conference on Machine Learning}, pages 261--268, San
  Francisco, CA. Morgan Kaufmann.
\newblock An expanded version was published as Technical Report CMU-CS-95-103,
  Carnegie Mellon University, Pittsburgh, PA, 1995.

\bibitem[Gordon, 1996]{Gordon-NIPS8}
Gordon, G.~J. (1996).
\newblock Stable fitted reinforcement learning.
\newblock In D.~S.~Touretzky, M. C.~Mozer, M. E.~H., editor, {\em Advances in
  Neural Information Processing Systems: {P}roceedings of the 1995 Conference},
  pages 1052--1058, Cambridge, MA. MIT Press.

\bibitem[Griffith, 1966]{Griffith-66}
Griffith, A.~K. (1966).
\newblock A new machine learning technique applied to the game of checkers.
\newblock Technical Report Project {MAC} Artificial Intelligence Memo 94,
  Massachusetts Institute of Technology.

\bibitem[Griffith, 1974]{Griffith-74}
Griffith, A.~K. (1974).
\newblock A comparison and evaluation of three machine learning procedures as
  applied to the game of checkers.
\newblock {\em Artificial Intelligence}, 5:137--148.

\bibitem[Gullapalli, 1990]{Vijay-NN}
Gullapalli, V. (1990).
\newblock A stochastic reinforcement algorithm for learning real-valued
  functions.
\newblock {\em Neural Networks}, 3:671--692.

\bibitem[Gurvits et~al., 1994]{Gurvits-Lin-Hanson-94}
Gurvits, L., Lin, L.-J., and Hanson, S.~J. (1994).
\newblock Incremental learning of evaluation functions for absorbing {M}arkov
  chains: {N}ew methods and theorems.
\newblock Preprint.

\bibitem[Hampson, 1983]{Hampson-thesis}
Hampson, S.~E. (1983).
\newblock {\em A Neural Model of Adaptive Behavior}.
\newblock PhD thesis, University of California, Irvine, CA.

\bibitem[Hampson, 1989]{Hampson-book}
Hampson, S.~E. (1989).
\newblock {\em Connectionist Problem Solving: {C}omputational Aspects of
  Biological Learning}.
\newblock Birkhauser, Boston.

\bibitem[Hawkins and Kandel, 1984]{Hawkins-Kandel}
Hawkins, R.~D. and Kandel, E.~R. (1984).
\newblock Is there a cell-biological alphabet for simple forms of learning?
\newblock {\em Psychological Review}, 91:375--391.

\bibitem[Hersh and Griego, 1969]{Hersh-Griego}
Hersh, R. and Griego, R.~J. (1969).
\newblock Brownian motion and potential theory.
\newblock {\em Scientific American}, pages 66--74.

\bibitem[Hilgard and Bower, 1975]{Hilgard-Bower}
Hilgard, E.~R. and Bower, G.~H. (1975).
\newblock {\em Theories of Learning}.
\newblock Prentice-Hall, Englewood Cliffs, NJ.

\bibitem[Hinton, 1984]{Hinton-coding}
Hinton, G.~E. (1984).
\newblock Distributed representations.
\newblock Technical Report CMU-CS-84-157, Department of Computer Science,
  Car\-neg\-ie-Mel\-lon University, Pittsburgh, PA.

\bibitem[Hochreiter and Schmidhuber, 1997]{Hochreiter-Schmidhuber}
Hochreiter, S. and Schmidhuber, J. (1997).
\newblock Long short-term memory.
\newblock {\em Neural Computation}.

\bibitem[Holland, 1975]{Holland-book}
Holland, J.~H. (1975).
\newblock {\em Adaptation in Natural and Artificial Systems}.
\newblock University of Michigan Press, Ann Arbor.

\bibitem[Holland, 1976]{Holland-76}
Holland, J.~H. (1976).
\newblock Adaptation.
\newblock In Rosen, R. and Snell, F.~M., editors, {\em Progress in Theoretical
  Biology}, volume~4, pages 263--293. Academic Press, NY.

\bibitem[Holland, 1986]{Holland}
Holland, J.~H. (1986).
\newblock Escaping brittleness: {T}he possibility of general-purpose learning
  algorithms applied to rule-based systems.
\newblock In Michalski, R.~S., Carbonell, J.~G., and Mitchell, T.~M., editors,
  {\em Machine Learning: {A}n Artificial Intelligence Approach, Volume II},
  pages 593--623. Morgan Kaufmann, San Mateo, CA.

\bibitem[Houk et~al., 1995]{Houk-Adams-Barto}
Houk, J.~C., Adams, J.~L., and Barto, A.~G. (1995).
\newblock A model of how the basal ganglia generates and uses neural signals
  that predict reinforcement.
\newblock In Houk, J.~C., Davis, J.~L., and Beiser, D.~G., editors, {\em Models
  of Information Processing in the Basal Ganglia}, pages 249--270. MIT Press,
  Cambridge, MA.

\bibitem[Howard, 1960]{Howard}
Howard, R. (1960).
\newblock {\em Dynamic Programming and {M}arkov Processes}.
\newblock MIT Press, Cambridge, MA.

\bibitem[Hull, 1943]{Hull-43}
Hull, C.~L. (1943).
\newblock {\em Principles of Behavior}.
\newblock D. Appleton-Century, NY.

\bibitem[Hull, 1952]{Hull-52}
Hull, C.~L. (1952).
\newblock {\em A Behavior System}.
\newblock Wiley, NY.

\bibitem[Jaakkola et~al., 1994]{Jaakkola-Jordan-Singh-NC}
Jaakkola, T., Jordan, M.~I., and Singh, S.~P. (1994).
\newblock On the convergence of stochastic iterative dynamic programming
  algorithms.
\newblock {\em Neural Computation}, 6.

\bibitem[Jaakkola et~al., 1995]{Jaakkola-Singh-Jordan-NIPS7}
Jaakkola, T., Singh, S.~P., and Jordan, M.~I. (1995).
\newblock Reinforcement learning algorithm for partially observable {M}arkov
  decision problems.
\newblock In G.~Tesauro, D.~Touretzky, T.~L., editor, {\em Advances in Neural
  Information Processing Systems: {P}roceedings of the 1994 Conference}, pages
  345--352, San Mateo, CA. Morgan Kaufmann.

\bibitem[Kaelbling, 1996]{Kaelbling-MLspecial-issue}
Kaelbling (1996).
\newblock A special issue of machine learning on reinforcement learning.
\newblock 22.

\bibitem[Kaelbling, 1993a]{Kaelbling-ICML93}
Kaelbling, L. (1993a).
\newblock Hierarchical learning in stochastic domains: {P}reliminary results.
\newblock In {\em Proceedings of the Tenth International Conference on Machine
  Learning}, pages 167--173. Morgan Kaufmann.

\bibitem[Kaelbling, 1993b]{Kaelbling-book}
Kaelbling, L.~P. (1993b).
\newblock {\em Learning in Embedded Systems}.
\newblock MIT Press, Cambridge MA.

\bibitem[Kaelbling et~al., 1996]{Kaelbling-etal-RLreview}
Kaelbling, L.~P., Littman, M.~L., and Moore, A.~W. (1996).
\newblock Reinforcement learning: {A} survey.
\newblock {\em Journal of Artificial Intelligence Research}, 4.

\bibitem[Kakutani, 1945]{Kakutani}
Kakutani, S. (1945).
\newblock {M}arkov processes and the dirichlet problem.
\newblock {\em Proc.\ Jap.\ Acad.}, 21:227--233.

\bibitem[Kalos and Whitlock, 1986]{Kalos-Whitlock}
Kalos, M.~H. and Whitlock, P.~A. (1986).
\newblock {\em Monte Carlo Methods}.
\newblock Wiley, NY.

\bibitem[Kanerva, 1988]{Kanerva-book}
Kanerva, P. (1988).
\newblock {\em Sparse Distributed Memory}.
\newblock MIT Press, Cambridge, MA.

\bibitem[Kanerva, 1993]{Kanerva-93}
Kanerva, P. (1993).
\newblock Sparse distributed memory and related models.
\newblock In Hassoun, M.~H., editor, {\em Associative Neural Memories: {T}heory
  and Implementation}, pages 50--76. Oxford University Press, NY.

\bibitem[Kashyap et~al., 1970]{Kashyap-Blaydon-Fu}
Kashyap, R.~L., Blaydon, C.~C., and Fu, K.~S. (1970).
\newblock Stochastic approximation.
\newblock In Mendel, J.~M. and Fu, K.~S., editors, {\em Adaptive, Learning, and
  Pattern Recognition Systems: {T}heory and Applications}. Academic Press, New
  York.

\bibitem[Keerthi and Ravindran, 1997]{Keerthi-Ravindran-handbook}
Keerthi, S.~S. and Ravindran, B. (1997).
\newblock Reinforcement learning.
\newblock In Fiesler, E. and Beale, R., editors, {\em Handbook of Neural
  Computation}. Oxford University Press, USA.

\bibitem[Kimble, 1961]{Kimble}
Kimble, G.~A. (1961).
\newblock {\em Hilgard and Marquis' Contitioning and Learning}.
\newblock Appleton-Century-Crofts, Inc., New York.

\bibitem[Kimble, 1967]{Kimble-foundations}
Kimble, G.~A. (1967).
\newblock {\em Foundations of Conditioning and Learning}.
\newblock Appleton-Century-Crofts.

\bibitem[Kirkpatrick et~al., 1983]{simulated-annealing}
Kirkpatrick, S., Gelatt, C.~D., and Vecchi, M.~P. (1983).
\newblock Optimization by simulated annealing.
\newblock {\em Science}, 220:671--680.

\bibitem[Klopf, 1972]{Klopf-72}
Klopf, A.~H. (1972).
\newblock Brain function and adaptive systems---{A} heterostatic theory.
\newblock Technical Report AFCRL-72-0164, Air Force Cambridge Research
  Laboratories, Bedford, MA.
\newblock A summary appears in {\it Proceedings of the International Conference
  on Systems, Man, and Cybernetics}, 1974, {IEEE} Systems, Man, and Cybernetics
  Society, Dallas, TX.

\bibitem[Klopf, 1975]{Klopf-SIGART}
Klopf, A.~H. (1975).
\newblock A comparison of natural and artificial intelligence.
\newblock {\em SIGART Newsletter}, 53:11--13.

\bibitem[Klopf, 1982]{Klopf-book}
Klopf, A.~H. (1982).
\newblock {\em The Hedonistic Neuron: {A} Theory of Memory, Learning, and
  Intelligence}.
\newblock Hemisphere, Washington, D.C.

\bibitem[Klopf, 1988]{Klopf-88}
Klopf, A.~H. (1988).
\newblock A neuronal model of classical conditioning.
\newblock {\em Psychobiology}, 16:85--125.

\bibitem[Kohonen, 1977]{Kohonen-77}
Kohonen, T. (1977).
\newblock {\em Associative Memory: {A} System Theoretic Approach}.
\newblock Springer-Varlag, Berlin.

\bibitem[Korf, 1988]{Korf-88}
Korf, R.~E. (1988).
\newblock Optimal path finding algorithms.
\newblock In Kanal, L.~N. and Kumar, V., editors, {\em Search in Artificial
  Intelligence}, pages 223--267. Springer Verlag, Berlin.

\bibitem[Kraft and Campagna, 1990]{Kraft-Campagna}
Kraft, L.~G. and Campagna, D.~P. (1990).
\newblock A summary comparison of {CMAC} neural network and traditional
  adaptive control systems.
\newblock In Miller, T., Sutton, R.~S., and Werbos, P.~J., editors, {\em Neural
  Networks for Control}, pages 143--169. MIT Press, Cambridge, MA.

\bibitem[Kraft et~al., 1992]{Kraft-Miller-Dietz}
Kraft, L.~G., Miller, W.~T., and Dietz, D. (1992).
\newblock Development and application of {CMAC} neural network-based control.
\newblock In White, D.~A. and Sofge, D.~A., editors, {\em Handbook of
  Intelligent Control: Neural, Fuzzy, and Adaptive Approaches}, pages 215--232.
  Van Nostrand Reinhold, New York.

\bibitem[Kuman and Varaiya, 1986]{Kumar-Varaiya}
Kuman, P.~R. and Varaiya, P. (1986).
\newblock {\em Stochastic Systems: {E}stimation, Identification, and Adaptive
  Control}.
\newblock Prentice-Hall, Englewood Cliffs, NJ.

\bibitem[Kumar, 1985]{Kumar-survey}
Kumar, P.~R. (1985).
\newblock A survey of some results in stochastic adaptive control.
\newblock {\em SIAM Journal of Control and Optimization}, 23:329--380.

\bibitem[Kumar and Kanal, 1988]{Kumar-Kanal}
Kumar, V. and Kanal, L.~N. (1988).
\newblock The {CDP}: {A} unifying formulation for heuristic search, dynamic
  programming, and branch-and-bound.
\newblock In Kanal, L.~N. and Kumar, V., editors, {\em Search in Artificial
  Intelligence}, pages 1--37. Springer-Verlag.

\bibitem[Kushner and Dupuis, 1992]{Kushner-Dupuis}
Kushner, H.~J. and Dupuis, P. (1992).
\newblock {\em Numerical Methods for Stochastic Control Problems in Continuous
  Time}.
\newblock Springer-Verlag, New York.

\bibitem[Lai, 1987]{Lai}
Lai, T.~L. (1987).
\newblock Adaptive treatment allocation and the multi-armed bandit problem.
\newblock {\em The Annals of Statistics}, 15(3):1091--1114.

\bibitem[Lang et~al., 1990]{Lang-etal-90}
Lang, K.~J., Waibel, A.~H., and Hinton, G.~E. (1990).
\newblock A time-delay neural network architecture for isolated word
  recognition.
\newblock {\em Neural Networks}, 3:33--43.

\bibitem[Lin and Kim, 1991]{Lin-Kim}
Lin, C.-S. and Kim, H. (1991).
\newblock Cmac-based adaptive critic self-learning control.
\newblock {\em IEEE Transactions on Neural Networks}, 2:530--533.

\bibitem[Lin, 1992]{Lin-ML92}
Lin, L.-J. (1992).
\newblock Self-improving reactive agents based on reinforcement learning,
  planning and teaching.
\newblock {\em Machine Learning}, 8:293--321.

\bibitem[Lin and Mitchell, 1992]{Lin-Mitchell}
Lin, L.-J. and Mitchell, T. (1992).
\newblock Reinforcement learning with hidden states.
\newblock In {\em Proceedings of the Second International Conference on
  Simulation of Adaptive Behavior: From Animals to Animats}, pages 271--280.
  MIT Press.

\bibitem[Littman, 1994]{Littman-multiagent}
Littman, M.~L. (1994).
\newblock {M}arkov games as a framework for multi-agent reinforcement learning.
\newblock In {\em Proceedings of the Eleventh International Conference on
  Machine Learning}, pages 157--163, San Francisco, CA. Morgan Kaufmann.

\bibitem[Littman et~al., 1995a]{Littman-etal-ml95}
Littman, M.~L., Cassandra, A.~R., and Kaelbling, L.~P. (1995a).
\newblock Learning policies for partially observable environments: {S}caling
  up.
\newblock In Prieditis, A. and Russell, S., editors, {\em Proceedings of the
  Twelfth International Conference on Machine Learning}, pages 362--370, San
  Francisco, CA. Morgan Kaufmann.

\bibitem[Littman et~al., 1995b]{Littman-etal-complexity}
Littman, M.~L., Dean, T.~L., and Kaelbling, L.~P. (1995b).
\newblock On the complexity of solving {M}arkov decision processes.
\newblock In {\em Proceedings of the Eleventh International Conference on
  Uncertainty in Artificial Intelligence}.

\bibitem[Ljung and {S\"{o}derstrom}, 1983]{Ljung-Soderstrom}
Ljung, L. and {S\"{o}derstrom}, T. (1983).
\newblock {\em Theory and Practice of Recursive Identification}.
\newblock MIT Press, Cambridge, MA.

\bibitem[Lovejoy, 1991]{Lovejoy-survey}
Lovejoy, W.~S. (1991).
\newblock A survey of algorithmic methods for partially observed {M}arkov
  decision processes.
\newblock {\em Annals of Operations Research}, 28:47--66.

\bibitem[Luce, 1959]{Luce-59}
Luce, D. (1959).
\newblock {\em Individual Choice Behavior}.
\newblock Wiley, NY.

\bibitem[M.~Zweben, 1994]{Zweben-etal-94}
M.~Zweben, B.~Daun, M.~D. (1994).
\newblock Scheduling and rescheduling with iterative repair.
\newblock In Zweben, M. and Fox, M.~S., editors, {\em Intelligent Scheduling},
  pages 241--255. Morgan Kaufmann, San Francisco, CA.

\bibitem[Maclin and Shavlik, 1994]{Maclin-Shavlik}
Maclin, R. and Shavlik, J.~W. (1994).
\newblock Incorporating advice into agents that learn from reinforcements.
\newblock In {\em Proceedings of the Twelfth National Conference on Artificial
  Intelligence (AAAI-94)}.

\bibitem[Mahadevan, 1996]{Mahadevan-ML96}
Mahadevan, S. (1996).
\newblock Average reward reinforcement learning: {F}oundations, algorithms, and
  empirical results.
\newblock {\em Machine Learning}, 22:159--196.

\bibitem[Markey, 1994]{Markey-94}
Markey, K.~L. (1994).
\newblock Efficient learning of multiple degree-of-freedom control problems
  with quasi-independent q-agents.
\newblock In Mozer, M.~C., Smolensky, P., Touretzky, D.~S., Elman, J.~L., and
  Weigend, A.~S., editors, {\em Proceedings of the 1009 Connectionist Models
  Summer School}, Hillsdale, NJ. Erlbaum.

\bibitem[Mazur, 1994]{Mazur}
Mazur, J.~E. (1994).
\newblock {\em Learning and Behavior, Third Edition}.
\newblock Prentice-Hall, Englewood Cliffs, NJ.

\bibitem[Mc{C}allum, 1992]{McCallum-ml92}
Mc{C}allum, A.~K. (1992).
\newblock Reinforcement learning with perceptual aliasing: {T}he perceptual
  distinctions approach.
\newblock In {\em Proceedings of the Tenth National Conference on Artificial
  Intelligence}, pages 183--188, Menlo Park, CA. AAAI Press/MIT Press.

\bibitem[Mc{C}allum, 1993]{McCallum-ml93}
Mc{C}allum, A.~K. (1993).
\newblock Overcoming incomplete perception with utile distinction memory.
\newblock In {\em Proceedings of the Tenth International Conference on Machine
  Learning}, pages 190--196. Morgan Kaufmann.

\bibitem[Mc{C}allum, 1995]{McCallum-thesis}
Mc{C}allum, A.~K. (1995).
\newblock {\em Reinforcement Learning with Selective Perception and Hidden
  State}.
\newblock PhD thesis, University of Rochester, Rochester.

\bibitem[Mendel, 1966]{Mendel-66}
Mendel, J.~M. (1966).
\newblock Applications of artificial intelligence techniques to a spacecraft
  control problem.
\newblock Technical Report NASA CR-755, National Aeronautics and Space
  Administration.

\bibitem[Mendel and McLaren, 1970]{Mendel-McLaren}
Mendel, J.~M. and McLaren, R.~W. (1970).
\newblock Reinforcement learning control and pattern recognition systems.
\newblock In Mendel, J.~M. and Fu, K.~S., editors, {\em Adaptive, Learning and
  Pattern Recognition Systems: {T}heory and Applications}, pages 287--318.
  Academic Press, New York.

\bibitem[Michie, 1961]{Michie-61}
Michie, D. (1961).
\newblock Trial and error.
\newblock In Barnett, S.~A. and McLaren, A., editors, {\em Science Survey, Part
  2}, pages 129--145, Harmondsworth. Penguin.

\bibitem[Michie, 1963]{Michie-63}
Michie, D. (1963).
\newblock Experiments on the mechanisation of game learning. 1.
  characterization of the model and its parameters.
\newblock {\em Computer Journal}, 1:232--263.

\bibitem[Michie, 1974]{Michie-book}
Michie, D. (1974).
\newblock {\em On Machine Intelligence}.
\newblock Edinburgh University Press.

\bibitem[Michie and Chambers, 1968]{Michie-Chambers-68}
Michie, D. and Chambers, R.~A. (1968).
\newblock {BOXES}: {A}n experiment in adaptive control.
\newblock In Dale, E. and Michie, D., editors, {\em Machine Intelligence 2},
  pages 137--152. Oliver and Boyd.

\bibitem[Miller and Williams, 1992]{Miller-Williams}
Miller, S. and Williams, R.~J. (1992).
\newblock Learning to control a bioreactor using a neural net dyna-q system.
\newblock In {\em Proceedings of the Seventh Yale Workshop on Adaptive and
  Learning Systems}, pages 167--172, Center for Systems Science, Dunham
  Laboratory, Yale University.

\bibitem[Miller et~al., 1994]{Miller-biped}
Miller, W.~T., Scalera, S.~M., and Kim, A. (1994).
\newblock Neural network control of dynamic balance for a biped walking robot.
\newblock In {\em Proceedings of the Eighth Yale Workshop on Adaptive and
  Learning Systems}, pages 156--161, Dunham Laboratory, Yale University. Center
  for Systems Science.

\bibitem[Minsky, 1954]{Minsky-thesis}
Minsky, M.~L. (1954).
\newblock {\em Theory of Neural-Analog Reinforcement Systems and its
  Application to the Brain-Model Problem}.
\newblock PhD thesis, Princeton University.

\bibitem[Minsky, 1961]{Minsky-steps}
Minsky, M.~L. (1961).
\newblock Steps toward artificial intelligence.
\newblock {\em Proceedings of the Institute of Radio Engineers}, 49:8--30.
\newblock Reprinted in E. A. Feigenbaum and J. Feldman, editors, {\it Computers
  and Thought}. McGraw-Hill, New York, 406--450, 1963.

\bibitem[Minsky, 1967]{Minsky-machines}
Minsky, M.~L. (1967).
\newblock {\em Computation: {F}inite and Infinite Machines}.
\newblock Prentice Hall, Englewood Cliffs, NJ.

\bibitem[Montague et~al., 1996]{Montague-Dayan-Sejnowski-96}
Montague, P.~R., Dayan, P., and Sejnowski, T.~J. (1996).
\newblock A framework for mesencephalic dopamine systems based on predictive
  hebbian learning.
\newblock {\em Journal of Neuroscience}, 16:1936--1947.

\bibitem[Moore, 1990]{Moore-thesis}
Moore, A.~W. (1990).
\newblock {\em Efficient Memory-Based Learning for Robot Control}.
\newblock PhD thesis, University of Cambridge, Cambridge, UK.

\bibitem[Moore, 1994]{Moore-NIPS6}
Moore, A.~W. (1994).
\newblock The parti-game algorithm for variable resolution reinforcement
  learning in multidimensional spaces.
\newblock In Cohen, J.~D., Tesauro, G., and Alspector, J., editors, {\em
  Advances in Neural Information Processing Systems: {P}roceedings of the 1993
  Conference}, pages 711--718, San Francisco, CA. Morgan Kaufmann.

\bibitem[Moore and Atkeson, 1993]{prioritized-sweeping}
Moore, A.~W. and Atkeson, C.~G. (1993).
\newblock Prioritized sweeping: {R}einforcement learning with less data and
  less real time.
\newblock {\em Machine Learning}, 13:103--130.

\bibitem[Moore et~al., 1986]{Moore-etal}
Moore, J.~W., Desmond, J.~E., Berthier, N.~E., Blazis, E.~J., Sutton, R.~S.,
  and Barto, A.~G. (1986).
\newblock Simulation of the classically conditioned nictitating membrane
  response by a neuron-like adaptive element: {I}. {R}esponse topography,
  neuronal firing, and interstimulus intervals.
\newblock {\em Behavioural Brain Research}, 21:143--154.

\bibitem[Narendra and Thathachar, 1989]{Narendra-book}
Narendra, K. and Thathachar, M. A.~L. (1989).
\newblock {\em Learning Automata: {A}n Introduction}.
\newblock Prentice Hall, Englewood Cliffs, NJ.

\bibitem[Narendra and Thathachar, 1974]{Narendra-Thathachar}
Narendra, K.~S. and Thathachar, M. A.~L. (1974).
\newblock Learning au\-tom\-a\-ta---{A} survey.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics}, 4:323--334.

\bibitem[Nie and Haykin, 1996]{Nie-Haykin-96}
Nie, J. and Haykin, S. (1996).
\newblock A dynamic channel assignment policy through q-learning.
\newblock CRL Report 334, Hamilton, Ontario, Canada L8S 4K1.

\bibitem[Page, 1977]{Page-77}
Page, C.~V. (1977).
\newblock Heuristics for signature table analysis as a pattern recognition
  technique.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  SMC-7:77--86.

\bibitem[Parr and Russell, 1995]{Parr-Russell}
Parr, R. and Russell, S. (1995).
\newblock Approximating optimal policies for partially observable stochastic
  domains.
\newblock In {\em {P}roceedings of the Fourteenth International Joint
  Conference on Artificial Intelligence}.

\bibitem[Pavlov, 1927]{Pavlov}
Pavlov, P.~I. (1927).
\newblock {\em Conditioned Reflexes}.
\newblock Oxford, London.

\bibitem[Pearl, 1984]{Pearl}
Pearl, J. (1984).
\newblock {\em Heuristics: {I}ntelligent Search Strategies for Computer Problem
  Solving}.
\newblock Addison-Wesley.

\bibitem[Peng, 1993]{Peng-thesis}
Peng, J. (1993).
\newblock {\em Efficient Dynamic Programming-Based Learning for Control}.
\newblock PhD thesis, Northeastern University, Boston, MA.

\bibitem[Peng and Williams, 1993]{Peng-Williams}
Peng, J. and Williams, R.~J. (1993).
\newblock Efficient learning and planning within the {D}yna framework.
\newblock {\em Adaptive Behavior}, 1(4).

\bibitem[Peng and Williams, 1994]{Peng-Williams-94}
Peng, J. and Williams, R.~J. (1994).
\newblock Incremental multi-step q-learning.
\newblock In Cohen, W.~W. and Hirsh, H., editors, {\em Proceedings of the
  Eleventh International Conference on Machine Learning}, pages 226--232.

\bibitem[Peng and Williams, 1996]{Peng-Williams-96}
Peng, J. and Williams, R.~J. (1996).
\newblock Incremental multi-step q-learning.
\newblock {\em Machine Learning}, 22(1/2/3).

\bibitem[Phansalkar and Thathachar, 1995]{Phansalkar-Thathachar}
Phansalkar, V.~V. and Thathachar, M. A.~L. (1995).
\newblock Local and global optimization algorithms for generalized learning
  automata.
\newblock {\em Neural Computation}, 7:950--973.

\bibitem[Poggio and Girosi, 1989]{Poggio-Girosi}
Poggio, T. and Girosi, F. (1989).
\newblock A theory of networks for approximation and learning.
\newblock A.I. Memo 1140, Artificial Intelligence Laboratory, Massachusetts
  Institute of Technology.

\bibitem[Poggio and Girosi, 1990]{Poggio-Girosi-Science}
Poggio, T. and Girosi, F. (1990).
\newblock Regularization algorithms for learning that are equivalent to
  multilayer networks.
\newblock {\em Science}, 247:978--982.

\bibitem[Powell, 1987]{Powell-RBFs}
Powell, M. J.~D. (1987).
\newblock Radial basis functions for multivariate interpolation: {A} review.
\newblock In Mason, J.~C. and Cox, M.~G., editors, {\em Algorithms for
  Approximation}. Clarendon Press, Oxford.

\bibitem[Puterman, 1994]{Puterman-book}
Puterman, M.~L. (1994).
\newblock {\em {M}arkov Decision Problems}.
\newblock Wiley, NY.

\bibitem[Puterman and Shin, 1978]{Puterman-Shin}
Puterman, M.~L. and Shin, M.~C. (1978).
\newblock Modified policy iteration algorithms for discounted {M}arkov decision
  problems.
\newblock {\em Management Science}, 24:1127--1137.

\bibitem[Reetz, 1977]{Reetz}
Reetz, D. (1977).
\newblock Approximate solutions of a discounted {M}arkovian decision process.
\newblock {\em Bonner Mathematische Schriften, vol 98: Dynamische Optimierung},
  pages 77--92.

\bibitem[Ring, 1994]{Ring-1994}
Ring, M.~B. (1994).
\newblock {\em Continual Learning in Reinforcement Environments}.
\newblock PhD thesis, University of Texas at Austin, Austin, Texas 78712.

\bibitem[Rivest and Schapire, 1987]{Rivest-Schapire-87}
Rivest, R.~L. and Schapire, R.~E. (1987).
\newblock Diversity-based inference of finite automata.
\newblock In {\em Proceedings of the Twenty-Eighth Annual Symposium on
  Foundations of Computer Science}, pages 78--87.

\bibitem[Robbins, 1952]{Robbins}
Robbins, H. (1952).
\newblock Some aspects of the sequential design of experiments.
\newblock {\em Bulletin of the American Mathematical Society}, 58:527--535.

\bibitem[Robertie, 1992]{Robertie-92}
Robertie, B. (1992).
\newblock Carbon versus silicon: {M}atching wits with {TD}-gammon.
\newblock {\em Inside Backgammon}, 2(2):14--22.

\bibitem[Rosenblatt, 1961]{Rosenblatt}
Rosenblatt, F. (1961).
\newblock {\em Principles of Neurodynamics: {P}erceptrons and the Theory of
  Brain Mechanisms}.
\newblock Spartan Books, 6411 Chillum Place N.W., Washington, D.C.

\bibitem[Ross, 1983]{Ross}
Ross, S. (1983).
\newblock {\em Introduction to Stochastic Dynamic Programming}.
\newblock Academic Press, New York.

\bibitem[Rubinstein, 1981]{Rubinstein}
Rubinstein, R.~Y. (1981).
\newblock {\em Simulation and the Monte Carlo Method}.
\newblock Wiley, NY.

\bibitem[Rumelhart et~al., 1986]{Rumelhart-Hinton-Williams}
Rumelhart, D.~E., Hinton, G.~E., and Williams, R.~J. (1986).
\newblock Learning internal representations by error propagation.
\newblock In Rumelhart, D.~E. and McClelland, J.~L., editors, {\em Parallel
  Distributed Processing: {E}xplorations in the Microstructure of Cognition,
  vol.1: {F}oundations}. Bradford Books/MIT Press, Cambridge, MA.

\bibitem[Rummery, 1995]{Rummery-thesis}
Rummery, G.~A. (1995).
\newblock {\em Problem Solving with Reinforcement Learning}.
\newblock PhD thesis, Cambridge University.

\bibitem[Rummery and Niranjan, 1994]{Rummery-Niranjan}
Rummery, G.~A. and Niranjan, M. (1994).
\newblock On-line q-learning using connectionist systems.
\newblock Technical Report CUED/F-INFENG/TR 166, Cambridge University
  Engineering Department.

\bibitem[Russell and Norvig, 1995]{Russell-Norvig}
Russell, S. and Norvig, P. (1995).
\newblock {\em Artificial Intelligence: {A} Modern Approach}.
\newblock Prentice Hall, Englewood Cliffs, NJ.

\bibitem[Rust, 1996]{Rust}
Rust, J. (1996).
\newblock Numerical dynamic programming in economics.
\newblock In Amman, H., Kendrick, D., and Rust, J., editors, {\em Handbook of
  Computational Economics}, pages 614--722. Elsevier, Amsterdam.

\bibitem[S.~J.~Bradtke, 1994]{Bradtke-Ydstie-Barto-ACC94}
S.~J.~Bradtke, B. E.~Ydstie, A. G.~B. (1994).
\newblock Adaptive linear quadratic control using policy iteration.
\newblock In {\em Proceedings of the American Control Conference}.

\bibitem[Samuel, 1959]{Samuel-59}
Samuel, A.~L. (1959).
\newblock Some studies in machine learning using the game of checkers.
\newblock {\em IBM Journal on Research and Development}, pages 210--229.
\newblock Reprinted in E. A. Feigenbaum and J. Feldman, editors, {\it Computers
  and Thought}, McGraw-Hill, New York, 1963.

\bibitem[Samuel, 1967]{Samuel}
Samuel, A.~L. (1967).
\newblock Some studies in machine learning using the game of checkers.
  {II}---{R}ecent progress.
\newblock {\em IBM Journal on Research and Development}, pages 601--617.

\bibitem[Schultz and Melsa, 1967]{Schultz-Melsa}
Schultz, D.~G. and Melsa, J.~L. (1967).
\newblock {\em State Functions and Linear Control Systems}.
\newblock McGraw-Hill, New York.

\bibitem[Schultz et~al., 1997]{Schultz-Dayan-Montague}
Schultz, W., Dayan, P., and Montague, P.~R. (1997).
\newblock A neural substrate of prediction and reward.
\newblock {\em Science}, 275:1593--1598.

\bibitem[Schwartz, 1993]{Schwartz}
Schwartz, A. (1993).
\newblock A reinforcement learning method for maximizing undiscounted rewards.
\newblock In {\em Proceedings of the Tenth International Conference on Machine
  Learning}, pages 298--305. Morgan Kaufmann.

\bibitem[Schweitzer and Seidmann, 1985]{Schweitzer-Seidmann}
Schweitzer, P.~J. and Seidmann, A. (1985).
\newblock Generalized polynomial approximations in {M}arkovian decision
  processes.
\newblock {\em Journal of Mathematical Analysis and Applications},
  110:568--582.

\bibitem[Selfridge et~al., 1985]{tracking-training}
Selfridge, O.~J., Sutton, R.~S., and Barto, A.~G. (1985).
\newblock Training and tracking in robotics.
\newblock In Joshi, A., editor, {\em Proceedings of the Ninth International
  Joint Conference of Artificial Intelligence}, pages 670--672, San Mateo, CA.
  Morgan Kaufmann.

\bibitem[Shannon, 1950a]{Shannon-SA50}
Shannon, C.~E. (1950a).
\newblock A chess-playing machine.
\newblock {\em Scientific American}, 182:48--51.

\bibitem[Shannon, 1950b]{Shannon-PM50}
Shannon, C.~E. (1950b).
\newblock Programming a computer for playing chess.
\newblock {\em Philosophical Magazine}, 41:256--275.

\bibitem[Shewchuk and Dean, 1990]{Shewchuk-Dean-90}
Shewchuk, J. and Dean, T. (1990).
\newblock Towards learning time-varying functions with high input
  dimensionality.
\newblock In {\em Proceedings of the Fifth {IEEE} International Symposium on
  Intelligent Control}, pages 383--388. IEEE.

\bibitem[Singh, 1992a]{Singh-AAAI92}
Singh, S.~P. (1992a).
\newblock Reinforcement learning with a hierarchy of abstract models.
\newblock In {\em Proceedings of the Tenth National Conference on Artificial
  Intelligence}, pages 202--207, Menlo Park, CA. AAAI Press/MIT Press.

\bibitem[Singh, 1992b]{Singh-ML92}
Singh, S.~P. (1992b).
\newblock Scaling reinforcement learning algorithms by learning variable
  temporal resolution models.
\newblock In {\em Proceedings of the Ninth International Machine Learning
  Conference}, pages 406--415, San Mateo, CA. Morgan Kaufmann.

\bibitem[Singh, 1993]{Singh-thesis}
Singh, S.~P. (1993).
\newblock {\em Learning to Solve {M}arkovian Decision Processes}.
\newblock PhD thesis, University of Massachusetts, Amherst.
\newblock Appeared as CMPSCI Technical Report 93-77.

\bibitem[Singh and Bertsekas, 1997]{Singh-Bertsekas-NIPS9}
Singh, S.~P. and Bertsekas, D. (1997).
\newblock Reinforcement learning for dynamic channel allocation in cellular
  telephone systems.
\newblock In {\em Advances in Neural Information Processing Systems:
  {P}roceedings of the 1996 Conference}, Cambridge, MA. MIT Press.

\bibitem[Singh et~al., 1994]{Singh-Jaakkola-Jordan-POMDP}
Singh, S.~P., Jaakkola, T., and Jordan, M.~I. (1994).
\newblock Learning without state-estimation in partially observable {M}arkovian
  decision problems.
\newblock In Cohen, W.~W. and Hirsch, H., editors, {\em Proceedings of the
  Eleventh International Conference on Machine Learning}, pages 284--292, San
  Francisco, CA. Morgan Kaufmann.

\bibitem[Singh et~al., 1995]{Singh-Jaakkola-Jordan-softagg}
Singh, S.~P., Jaakkola, T., and Jordan, M.~I. (1995).
\newblock Reinforcement learing with soft state aggregation.
\newblock In G.~Tesauro, D.~Touretzky, T.~L., editor, {\em Advances in Neural
  Information Processing Systems: {P}roceedings of the 1994 Conference}, pages
  359--368, Cambridge, MA. MIT Press.

\bibitem[Singh and Sutton, 1996]{Singh-Sutton-ML96}
Singh, S.~P. and Sutton, R.~S. (1996).
\newblock Reinforcement learning with replacing eligibility traces.
\newblock {\em Machine Learning}, 22:123--158.

\bibitem[Sivarajan et~al., 1990]{MAXAVAIL}
Sivarajan, K.~N., McEliece, R.~J., and Ketchum, J.~W. (1990).
\newblock Dynamic channel assignment in cellular radio.
\newblock In {\em Proceedings of the 40th Vehicular Technology Conference},
  pages 631--637.

\bibitem[Skinner, 1938]{Skinner-38}
Skinner, B.~F. (1938).
\newblock {\em The Behavior of Organisms}.
\newblock Appleton-Century, NY.

\bibitem[Sofge and White, 1992]{Sofge-White-handbook}
Sofge, D.~A. and White, D.~A. (1992).
\newblock Applied learning: {O}ptimal control for manufacturing.
\newblock In White, D.~A. and Sofge, D.~A., editors, {\em Handbook of
  Intelligent Control: Neural, Fuzzy, and Adaptive Approaches}, pages 259--281.
  Van Nostrand Reinhold, New York.

\bibitem[Spong, 1994]{Spong}
Spong, M.~W. (1994).
\newblock Swing up control of the acrobot.
\newblock In {\em Proceedings of the 1994 {IEEE} Conference on Robotics and
  Automation}, San Diego, CA.

\bibitem[Staddon, 1983]{Staddon-book}
Staddon, J. E.~R. (1983).
\newblock {\em Adaptive Behavior and Learning}.
\newblock Cambridge University Press, Cambridge.

\bibitem[Sutton, 1978a]{Sutton-78b}
Sutton, R.~S. (1978a).
\newblock Learning theory support for a single channel theory of the brain.

\bibitem[Sutton, 1978b]{Sutton-78a}
Sutton, R.~S. (1978b).
\newblock Single channel theory: {A} neuronal theory of learning.
\newblock {\em Brain Theory Newsletter}, 4:72--75.

\bibitem[Sutton, 1978c]{Sutton-78c}
Sutton, R.~S. (1978c).
\newblock A unified theory of expectation in classical and instrumental
  conditioning.

\bibitem[Sutton, 1984]{Sutton-thesis}
Sutton, R.~S. (1984).
\newblock {\em Temporal Credit Assignment in Reinforcement Learning}.
\newblock PhD thesis, University of Massachusetts, Amherst, MA.

\bibitem[Sutton, 1988]{Sutton-ML}
Sutton, R.~S. (1988).
\newblock Learning to predict by the method of temporal differences.
\newblock {\em Machine Learning}, 3:9--44.

\bibitem[Sutton, 1990]{Sutton-ML90}
Sutton, R.~S. (1990).
\newblock Integrated architectures for learning, planning, and reacting based
  on approximating dynamic programming.
\newblock In {\em Proceedings of the Seventh International Conference on
  Machine Learning}, pages 216--224, San Mateo, CA. Morgan Kaufmann.

\bibitem[Sutton, 1991a]{Sutton-AAAISS91}
Sutton, R.~S. (1991a).
\newblock Dyna, an integrated architecture for learning, planning, and
  reacting.
\newblock {\em SIGART Bulletin}, 2:160--163.
\newblock Also appeared in {\em Working Notes of the 1991 AAAI Spring
  Symposium}, pages 151--155.

\bibitem[Sutton, 1991b]{Sutton-ML91}
Sutton, R.~S. (1991b).
\newblock Planning by incremental dynamic programming.
\newblock In Birnbaum, L.~A. and Collins, G.~C., editors, {\em Proceedings of
  the Eighth International Workshop on Machine Learning}, pages 353--357, San
  Mateo, CA. Morgan Kaufmann.

\bibitem[Sutton, 1992]{Sutton-MLspecial_issue}
Sutton, R.~S., editor (1992).
\newblock {\em A Special Issue of Machine Learning on Reinforcement Learning},
  volume~8. { \it Machine Learning}.
\newblock Also published as {\it Reinforcement Learnng}, Kluwer Academic Press,
  Boston, MA 1992.

\bibitem[Sutton, 1995]{Sutton-ml95}
Sutton, R.~S. (1995).
\newblock {TD} models: {M}odeling the world at a mixture of time scales.
\newblock In Prieditis, A. and Russell, S., editors, {\em Proceedings of the
  Twelfth International Conference on Machine Learning}, pages 531--539, San
  Francisco, CA. Morgan Kaufmann.

\bibitem[Sutton, 1996]{Sutton-NIPS95}
Sutton, R.~S. (1996).
\newblock Generalization in reinforcement learning: {S}uccessful examples using
  sparse coarse coding.
\newblock In Touretzky, D.~S., Mozer, M.~C., and Hasselmo, M.~E., editors, {\em
  Advances in Neural Information Processing Systems: {P}roceedings of the 1995
  Conference}, pages 1038--1044, Cambridge, MA. MIT Press.

\bibitem[Sutton and Barto, 1981a]{internal-model}
Sutton, R.~S. and Barto, A.~G. (1981a).
\newblock An adaptive network that constructs and uses an internal model of its
  world.
\newblock {\em Cognition and Brain Theory}, 3:217--246.

\bibitem[Sutton and Barto, 1981b]{Sutton-Barto}
Sutton, R.~S. and Barto, A.~G. (1981b).
\newblock Toward a modern theory of adaptive networks: {E}xpectation and
  prediction.
\newblock {\em Psychological Review}, 88:135--170.

\bibitem[Sutton and Barto, 1987]{Sutton-Barto-Cogsci87}
Sutton, R.~S. and Barto, A.~G. (1987).
\newblock A temporal-difference model of classical conditioning.
\newblock In {\em Proceedings of the Ninth Annual Conference of the Cognitive
  Science Society}, Hillsdale, NJ. Erlbaum.

\bibitem[Sutton and Barto, 1990]{Sutton-Barto-90}
Sutton, R.~S. and Barto, A.~G. (1990).
\newblock Time-derivative models of pavlovian reinforcement.
\newblock In Gabriel, M. and Moore, J., editors, {\em Learning and
  Computational Neuroscience: {F}oundations of Adaptive Networks}, pages
  497--537. MIT Press, Cambridge, MA.

\bibitem[Sutton and Pinette, 1985]{Sutton-Pinette-Cogsci85}
Sutton, R.~S. and Pinette, B. (1985).
\newblock The learning of world models by connectionist networks.
\newblock In {\em Proceedings of the Seventh Annual Conference of the Cognitive
  Science Society}, Irvine, CA.

\bibitem[Sutton and Singh, 1994]{Sutton-Singh-94}
Sutton, R.~S. and Singh, S. (1994).
\newblock On bias and step size in temporal-difference learning.
\newblock In {\em Proceedings of the Eighth Yale Workshop on Adaptive and
  Learning Systems}, pages 91--96, New Haven, CT. Yale University.

\bibitem[Tadepally and Ok, 1994]{Tadepalli-Ok}
Tadepally, P. and Ok, D. (1994).
\newblock H-learning: {A} reinforcement learning method to optimize
  undiscounted average reward.
\newblock Technical Report 94-30-01, Oregon State University.

\bibitem[Tan, 1991]{Tan-ML91}
Tan, M. (1991).
\newblock Learning a cost-sensitive internal representation for reinforcement
  learning.
\newblock In Birnbaum, L.~A. and Collins, G.~C., editors, {\em Proceedings of
  the Eighth International Workshop on Machine Learning}, pages 358--362, San
  Mateo, CA. Morgan Kaufmann.

\bibitem[Tan, 1993]{Tan-93}
Tan, M. (1993).
\newblock Multi-agent reinforcement learning: {I}ndependent vs.\ cooperative
  agents.
\newblock In {\em Proceedings of the Tenth International Conference on Machine
  Learning}, pages 330--337. Morgan Kaufmann.

\bibitem[Tesauro, 1986]{Tesauro-CC}
Tesauro, G.~J. (1986).
\newblock Simple neural models of classical conditioning.
\newblock {\em Biological Cybernetics}, 55:187--200.

\bibitem[Tesauro, 1992]{Tesauro-backgammon}
Tesauro, G.~J. (1992).
\newblock Practical issues in temporal difference learning.
\newblock {\em Machine Learning}, 8:257--277.

\bibitem[Tesauro, 1994]{Tesauro-NC}
Tesauro, G.~J. (1994).
\newblock {TD}--gammon, a self-teaching backgammon program, achieves
  master-level play.
\newblock {\em Neural Computation}, 6(2):215--219.

\bibitem[Tesauro, 1995]{Tesauro-ACM}
Tesauro, G.~J. (1995).
\newblock Temporal difference learning and {TD}-{G}ammon.
\newblock {\em Communications of the ACM}, 38:58--68.

\bibitem[Tesauro and Galperin, 1997]{Tesauro-Galperin-NIPS97}
Tesauro, G.~J. and Galperin, G.~R. (1997).
\newblock On-line policy improvement using monte-carlo search.
\newblock In {\em Advances in Neural Information Processing Systems:
  {P}roceedings of the 1996 Conference}, Cambridge, MA. MIT Press.

\bibitem[Tham, 1994]{Tham-thesis}
Tham, C.~K. (1994).
\newblock {\em Modular On-Line Function Approximation for Scaling up
  Reinforcement Learning}.
\newblock PhD thesis, Cambridge University.

\bibitem[Thathachar and Sastry, 1986]{Thathachar-Sastry-86}
Thathachar, M. A.~L. and Sastry, P.~S. (1986).
\newblock Estimator algorithms for learning automata.
\newblock In {\em Proceedings of the Platinum Jubilee Conference on Systems and
  Signal Processing}, Bengalore, India.

\bibitem[Thathachar and Sastry, 1995]{Thathachar-Sastry-85}
Thathachar, M. A.~L. and Sastry, P.~S. (1995).
\newblock A new approach to the design of reinforcement schemes for learning
  automata.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics},
  15:168--175.

\bibitem[Thompson, 1933]{Thompson-33}
Thompson, W.~R. (1933).
\newblock On the likelihood that one unknown probability exceeds another in
  view of the evidence of two samples.
\newblock {\em Biometrika}, 25:285--294.

\bibitem[Thompson, 1934]{Thompson-34}
Thompson, W.~R. (1934).
\newblock On the theory of apportionment.
\newblock {\em American Journal of Mathematics}, 57:450--457.

\bibitem[Thorndike, 1911]{Thorndike}
Thorndike, E.~L. (1911).
\newblock {\em Animal Intelligence}.
\newblock Hafner, Darien, Conn.

\bibitem[Thorp, 1966]{Thorp}
Thorp, E.~O. (1966).
\newblock {\em Beat the Dealer: {A} Winning Strategy for the Game of
  Twenty-One}.
\newblock Random House, New York.

\bibitem[Tolman, 1932]{Tolman}
Tolman, E.~C. (1932).
\newblock {\em Purposive Behavior in Animals and Men}.
\newblock Century, New York.

\bibitem[Tsetlin, 1973]{Tsetlin}
Tsetlin, M.~L. (1973).
\newblock {\em Automaton Theory and Modeling of Biological Systems}.
\newblock Academic Press, New York.

\bibitem[Tsitsiklis, 1994]{Tsitsiklis-ML94}
Tsitsiklis, J.~N. (1994).
\newblock Asynchronous stochastic approximation and q-learning.
\newblock {\em Machine Learning}, 16:185--202.

\bibitem[Tsitsiklis and {Van Roy}, 1996]{Tsitsiklis-VanRoy-ML96}
Tsitsiklis, J.~N. and {Van Roy}, B. (1996).
\newblock Feature-based methods for large scale dynamic programming.
\newblock {\em Machine Learning}, 22:59--94.

\bibitem[Tsitsiklis and {Van Roy}, 1997]{Tsitsiklis-VanRoy-97}
Tsitsiklis, J.~N. and {Van Roy}, B. (1997).
\newblock An analysis of temporal-difference learning with function
  approximation.
\newblock {\em IEEE Transactions on Automatic Control}.

\bibitem[Ungar, 1990]{Ungar-bioreactor}
Ungar, L.~H. (1990).
\newblock A bioreactor benchmark for adaptive network-based process control.
\newblock In Miller, W.~T., Sutton, R.~S., and Werbos, P.~J., editors, {\em
  Neural Networks for Control}, pages 387--402. MIT Press, Cambridge, MA.

\bibitem[Varga, 1962]{Varga-62}
Varga, R.~S. (1962).
\newblock {\em Matrix Iterative Analysis}.
\newblock Prentice-Hall, Englewood Cliffs, NJ.

\bibitem[Waltz and Fu, 1965]{Waltz-Fu}
Waltz, M.~D. and Fu, K.~S. (1965).
\newblock A heuristic approach to reinforcment learning control systems.
\newblock {\em IEEE Transactions on Automatic Control}, 10:390--398.

\bibitem[Watkins, 1989]{Watkins-thesis}
Watkins, C. J. C.~H. (1989).
\newblock {\em Learning from Delayed Rewards}.
\newblock PhD thesis, Cambridge University, Cambridge, England.

\bibitem[Watkins and Dayan, 1992]{Watkins-Dayan-ML}
Watkins, C. J. C.~H. and Dayan, P. (1992).
\newblock Q-learning.
\newblock {\em Machine Learning}, 8:279--292.

\bibitem[Werbos, 1992]{Werbos-handbook}
Werbos, P. (1992).
\newblock Approximate dynamic programming for real-time control and neural
  modeling.
\newblock In White, D.~A. and Sofge, D.~A., editors, {\em Handbook of
  Intelligent Control: Neural, Fuzzy, and Adaptive Approaches}, pages 493--525.
  Van Nostrand Reinhold, New York.

\bibitem[Werbos, 1977]{Werbos-77}
Werbos, P.~J. (1977).
\newblock Advanced forecasting methods for global crisis warning and models of
  intelligence.
\newblock {\em General Systems Yearbook}, 22:25--38.

\bibitem[Werbos, 1982]{Werbos-82}
Werbos, P.~J. (1982).
\newblock Applications of advances in nonlinear sensitivity analysis.
\newblock In Drenick, R.~F. and Kosin, F., editors, {\em System Modeling an
  Optimization}. Springer-Verlag.
\newblock Proceedings of the Tenth IFIP Conference, New York, 1981.

\bibitem[Werbos, 1987]{Werbos-SMC87}
Werbos, P.~J. (1987).
\newblock Building and understanding adaptive systems: {A}
  statistical/numerical approach to factory automation and brain research.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics}, pages
  7--20.

\bibitem[Werbos, 1988]{Werbos-NN}
Werbos, P.~J. (1988).
\newblock Generalization of back propagation with applications to a recurrent
  gas market model.
\newblock {\em Neural Networks}, 1:339--356.

\bibitem[Werbos, 1989]{Werbos-CDC}
Werbos, P.~J. (1989).
\newblock Neural networks for control and system identification.
\newblock In {\em Proceedings of the 28th Conference on Decision and Control},
  pages 260--265, Tampa, Florida.

\bibitem[Werbos, 1990]{Werbos-consistency}
Werbos, P.~J. (1990).
\newblock Consistency of {HDP} applied to simple reinforcement learning
  problem.
\newblock {\em Neural Networks}, 3:179--189.

\bibitem[White, 1969]{White}
White, D.~J. (1969).
\newblock {\em Dynamic Programming}.
\newblock Holden-Day, San Francisco.

\bibitem[White, 1985]{White1}
White, D.~J. (1985).
\newblock Real applications of {M}arkov decision processes.
\newblock {\em Interfaces}, 15:73--83.

\bibitem[White, 1988]{White2}
White, D.~J. (1988).
\newblock Further real applications of {M}arkov decision processes.
\newblock {\em Interfaces}, 18:55--61.

\bibitem[White, 1993]{White-survey}
White, D.~J. (1993).
\newblock A survey of applications of {M}arkov decision processes.
\newblock {\em Journal of the Operational Research Society}, 44:1073--1096.

\bibitem[Whitehead and Ballard, 1991]{Whitehead-Ballard-ML91}
Whitehead, S.~D. and Ballard, D.~H. (1991).
\newblock Learning to perceive and act by trial and error.
\newblock {\em Machine Learning}, 7(1):45--83.

\bibitem[Whitt, 1978]{Whitt}
Whitt, W. (1978).
\newblock Approximations of dynamic programs {I}.
\newblock {\em Mathematics of Operations Research}, 3:231--243.

\bibitem[Whittle, 1982]{Whittle1}
Whittle, P. (1982).
\newblock {\em Optimization over Time}, volume~1.
\newblock Wiley, NY.

\bibitem[Whittle, 1983]{Whittle2}
Whittle, P. (1983).
\newblock {\em Optimization over Time}, volume~2.
\newblock Wiley, NY.

\bibitem[Widrow et~al., 1973]{bootstrap}
Widrow, B., Gupta, N.~K., and Maitra, S. (1973).
\newblock Punish/reward: {L}earning with a critic in adaptive threshold
  systems.
\newblock {\em IEEE Transactions on Systems, Man, and Cybernetics}, 5:455--465.

\bibitem[Widrow and Hoff, 1960]{Widrow-Hoff}
Widrow, B. and Hoff, M.~E. (1960).
\newblock Adaptive switching circuits.
\newblock In {\em 1960 {WESCON} Convention Record Part {IV}}, pages 96--104.
\newblock Reprinted in J. A. Anderson and E. Rosenfeld, {\it Neurocomputing:
  {F}oundations of Research}, MIT Press, Cambridge, MA, 1988.

\bibitem[Widrow and Smith, 1964]{Widrow-Smith}
Widrow, B. and Smith, F.~W. (1964).
\newblock Pattern-recognizing control systems.
\newblock In {\em Computer and Information Sciences (COINS) Proceedings},
  Washington, D.C. Spartan.

\bibitem[Widrow and Stearns, 1985]{Widrow-Stearns}
Widrow, B. and Stearns, S.~D. (1985).
\newblock {\em Adaptive Signal Processing}.
\newblock Prentice-Hall, Inc., Englewood Cliffs, N.J.

\bibitem[Williams, 1986]{Williams-86}
Williams, R.~J. (1986).
\newblock Reinforcement learning in connectionist networks: {A} mathematical
  analysis.
\newblock Technical Report ICS 8605, Institute for Cognitive Science,
  University of California at San Diego, La Jolla, CA.

\bibitem[Williams, 1987]{Williams-87}
Williams, R.~J. (1987).
\newblock Reinforcement-learning connectionist systems.
\newblock Technical Report NU-CCS-87-3, College of Computer Science,
  Northeastern University, Boston, MA.

\bibitem[Williams, 1988]{Williams-ICNN88}
Williams, R.~J. (1988).
\newblock On the use of backpropagation in associative reinforcement learning.
\newblock In {\em Proceedings of the {IEEE} International Conference on Neural
  Networks}, pages 263--270, San Diego, CA.

\bibitem[Williams, 1992]{williams-REINFORCE}
Williams, R.~J. (1992).
\newblock Simple statistical gradient-following algorithms for connectionist
  reinforcement learning.
\newblock {\em Machine Learning}, 8:229--256.

\bibitem[Williams and Baird, 1990]{Williams-Baird-Yale90}
Williams, R.~J. and Baird, L.~C. (1990).
\newblock A mathematical analysis of actor-critic architectures for learning
  optimal controls through incremental dynamic programming.
\newblock In {\em Proceedings of the Sixth Yale Workshop on Adaptive and
  Learning Systems}, pages 96--101, New Haven, CT.

\bibitem[Wilson, 1994]{Wilson-94}
Wilson, S.~W. (1994).
\newblock {ZCS}: {A} zeroth order classifier system.
\newblock {\em Evolutionary Compuation}, 2:1--18.

\bibitem[Witten, 1976]{Witten-76}
Witten, I.~H. (1976).
\newblock The apparent conflict between estimation and control---{A} survey of
  the two-armed problem.
\newblock {\em Journal of the Franklin Institute}, 301:161--189.

\bibitem[Witten, 1977]{Witten}
Witten, I.~H. (1977).
\newblock An adaptive optimal controller for discrete-time {M}arkov
  environments.
\newblock {\em Information and Control}, 34:286--295.

\bibitem[Witten and Corbin, 1973]{Witten-Corbin}
Witten, I.~H. and Corbin, M.~J. (1973).
\newblock Human operators and automatic adaptive controllers: {A} comparative
  study on a particular control task.
\newblock {\em International Journal of Man-Machine Studies}, 5:75--104.

\bibitem[Yee et~al., 1990]{Yee-AAAI90}
Yee, R.~C., Saxena, S., Utgoff, P.~E., and Barto, A.~G. (1990).
\newblock Explaining temporal differences to create useful concepts for
  evaluating states.
\newblock In {\em Proceedings of the Eighth National Conference on Artificial
  Intelligence}, pages 882--888, Cambridge, MA.

\bibitem[Young, 1984]{Young}
Young, P. (1984).
\newblock {\em Recursive Estimation and Time-Series Analysis}.
\newblock Springer-Verlag.

\bibitem[Zhang and Yum, 1989]{Zhang-Yum-89}
Zhang, M. and Yum, T.~P. (1989).
\newblock Comparisons of channel-assignment strategies in cellular mobile
  telephone systems.
\newblock {\em {IEEE} Transactions on Vehicular Technology}, 38.

\bibitem[Zhang, 1996]{Zhang-thesis}
Zhang, W. (1996).
\newblock {\em Reinforcement Learning for Job-shop Scheduling}.
\newblock PhD thesis, Oregon State University.
\newblock Tech Report CS-96-30-1.

\bibitem[Zhang and Dietterich, 1995]{Zhang-Dietterich-IJCAI-95}
Zhang, W. and Dietterich, T.~G. (1995).
\newblock A reinforcement learning approach to job-shop scheduling.
\newblock In {\em Proceedings of the Fourteenth International Joint Conference
  on Artificial Intelligence}, pages 1114--1120.

\bibitem[Zhang and Dietterich, 1996]{Zhang-Dietterich-NIPS8}
Zhang, W. and Dietterich, T.~G. (1996).
\newblock High-performance job-shop scheduling with a time--delay
  {TD}$(\lambda)$ network.
\newblock In D.~S.~Touretzky, M. C.~Mozer, M. E.~H., editor, {\em Advances in
  Neural Information Processing Systems: {P}roceedings of the 1995 Conference},
  pages 1024--1030, Cambridge, MA. MIT Press.

\end{thebibliography}
