@inproceedings{6cf892c562d64e0c9fbe35bbedb00f20,
title = "Adaptive natural gradient learning algorithms for unnormalized statistical models",
abstract = "The natural gradient is a powerful method to improve the transient dynamics of learning by utilizing the geometric structure of the parameter space. Many natural gradient methods have been developed for maximum likelihood learning, which is based on Kullback-Leibler (KL) divergence and its Fisher metric. However, they require the computation of the normalization constant and are not applicable to statistical models with an analytically intractable normalization constant. In this study, we extend the natural gradient framework to divergences for the unnormalized statistical models: score matching and ratio matching. In addition, we derive novel adaptive natural gradient algorithms that do not require computationally demanding inversion of the metric and show their effectiveness in some numerical experiments. In particular, experimental results in a multi-layer neural network model demonstrate that the proposed method can escape from the plateau phenomena much faster than the conventional stochastic gradient descent method.",
keywords = "Multi-layer neural network, Natural gradient, Ratio matching, Score matching, Unnormalized statistical model",
author = "Ryo Karakida and Masato Okada and Amari, \{Shun Ichi\}",
note = "Publisher Copyright: {\textcopyright} Springer International Publishing Switzerland 2016.; 25th International Conference on Artificial Neural Networks, ICANN 2016 ; Conference date: 06-09-2016 Through 09-09-2016",
year = "2016",
doi = "10.1007/978-3-319-44778-0\_50",
language = "英語",
isbn = "9783319447773",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Verlag",
pages = "427--434",
editor = "Villa, \{Alessandro E.P.\} and Paolo Masulli and Rivero, \{Antonio Javier Pons\}",
booktitle = "Artificial Neural Networks and Machine Learning - 25th International Conference on Artificial Neural Networks, ICANN 2016, Proceedings",
address = "ドイツ",
}