@article {16, title = {Deep learning reaches the motor system}, journal = {Nature Methods}, volume = {15}, year = {2018}, month = {Jan-10-2018}, pages = {772 - 773}, type = {News and Views}, abstract = {

A new article by Pandarinath et al. describes an artificial neural network model that captures some key aspects of the activity of populations of neurons in the primary motor cortex.

}, issn = {1548-7091}, doi = {10.1038/s41592-018-0152-6}, url = {https://doi.org/10.1038/s41592-018-0152-6}, author = {Batista, Aaron P. and DiCarlo, James J.} } @article {32, title = {Deep Neural Networks Rival the Representation of Primate IT Cortex for Core Visual Object Recognition}, journal = {PLoS Computational Biology}, volume = {10}, year = {2014}, month = {12/2014}, pages = {e1003963}, abstract = {

The primate visual system achieves remarkable visual object recognition performance even in brief presentations, and under changes to object exemplar, geometric transformations, and background variation (a.k.a. core visual object recognition). This remarkable performance is mediated by the representation formed in inferior temporal (IT) cortex. In parallel, recent advances in machine learning have led to ever higher performing models of object recognition using artificial deep neural networks (DNNs). It remains unclear, however, whether the representational performance of DNNs rivals that of the brain. To accurately produce such a comparison, a major difficulty has been a unifying metric that accounts for experimental limitations, such as the amount of noise, the number of neural recording sites, and the number of trials, and computational limitations, such as the complexity of the decoding classifier and the number of classifier training examples. In this work, we perform a direct comparison that corrects for these experimental limitations and computational considerations. As part of our methodology, we propose an extension of \"kernel analysis\" that measures the generalization accuracy as a function of representational complexity. Our evaluations show that, unlike previous bio-inspired models, the latest DNNs rival the representational performance of IT cortex on this visual object recognition task. Furthermore, we show that models that perform well on measures of representational performance also perform well on measures of representational similarity to IT, and on measures of predicting individual IT multi-unit responses. Whether these DNNs rely on computational mechanisms similar to the primate visual system is yet to be determined, but, unlike all previous bio-inspired models, that possibility cannot be ruled out merely on representational performance grounds.

}, doi = {10.1371/journal.pcbi.1003963}, url = {https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003963\&type=printable}, author = {Cadieu, Charles F. and Hong, Ha and Yamins, Daniel L. K. and Pinto, Nicolas and Ardila, Diego and Solomon, Ethan A. and Majaj, Najib J. and DiCarlo, James J.}, editor = {Bethge, Matthias} } @inbook {49, title = {Do we have a strategy for understanding how the visual system accomplishes object recognition?}, booktitle = {Object Categorization: Computer and Human Vision Perspectives}, year = {2010}, publisher = {Cambridge University Press}, organization = {Cambridge University Press}, address = {New York, NY, USA}, issn = {978-0-521-88738-0}, author = {DiCarlo, James J.}, editor = {Dickenson, SJ and Leonardis, A. and Schiele, B. and Tarr, MJ.} } @proceedings {50, title = {Does the visual system use natural experience to construct size invariant object representations?}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2010}, month = {02/2010}, address = {Salt Lake City, Utah, USA}, abstract = {

Object recognition is challenging because each object produces myriad retinal images. Responses of neurons at the top of the ventral visual stream (inferior temporal cortex, IT) exhibit object selectivity that is unaffected by the image changes. How do IT neurons attain this tolerance (\"invariance\")? One powerful idea is that temporal contiguity of natural visual experience can instruct tolerance (e.g. Foldiak, Neural Computation, 1991): because objects remain present for many seconds, whereas object or viewer motion cause changes in each object\’s retinal image over shorter time intervals, the ventral stream could construct tolerance by learning to associate neuronal representations that occur closely in time. We recently found a neuronal signature of such learning in IT: temporally contiguous experience with different object images at different retinal positions can robustly reshape (\"break\") IT position tolerance, producing a tendency for IT neurons to confuse the identities of those temporally coupled objects across their manipulated positions (Li \& DiCarlo, Science, 2008). A similar manipulation can induce the same pattern of confusion in the position tolerance of human object perception (Cox, Meier, Oertelt, DiCarlo. Nat Neurosci, 2005). Does this IT neuronal learning reflect a canonical unsupervised learning algorithm the ventral stream relies on to achieve tolerance to all types of image variation (e.g. object size and pose changes)? To begin to answer this question, we here extend our position tolerance paradigm to object size changes. Unsupervised non-human primates were exposed to an altered visual world in which we temporally coupled the experience of two object images of different sizes at each animal\’s center of gaze: (e.g.) a small image of one object (P, neuronally preferred object) was consistently followed by a large image of a second object (N), rendering the small image of P temporally contiguous with the large image of N. We made IT neuronal selectivity measurements before and after the animals received ~2 hours of experience in the unsupervised, altered visual world. Consistent with our results on position tolerance, we found that this size experience manipulation robustly reshapes IT size tolerance over a period of hours. Specifically, unlike experienced controls, we found a change in neuronal selectivity (P-N) across the manipulated objects and their manipulated sizes, producing a tendency to confuse those object identities across those sizes. This change in size tolerance is specific to the manipulated objects, grew gradually stronger with increasing experience, and the rate of learning was similar to position tolerance learning (~5 spikes/s per hour of exposure). Finally, in a separate experiment, we examine how temporal direction of the experience affects the learning: do temporally-early images teach temporally-later ones, or vice-versa? We found greater learning for the temporally-later images, suggesting a Hebbian-like learning mechanism (e.g. Sprekeler \& Gerstner, COSYNE, 2009; Wallis \& Rolls, Prog Neurobiol, 1997). We speculate that these converging results on IT position and size tolerance plasticity reflect an underlying unsupervised cortical learning mechanism by which the ventral visual stream acquires and maintains its tolerant object representations.

}, doi = {10.3389/conf.fnins.2010.03.00326}, url = {https://www.frontiersin.org/Community/AbstractDetails.aspx?ABS_DOI=10.3389/conf.fnins.2010.03.00326\&eid=770\&sname=Computational_and_Systems_Neuroscience_2010}, author = {Li, Nuo and DiCarlo, James J.} } @article {68, title = {Does Learned Shape Selectivity in Inferior Temporal Cortex Automatically Generalize Across Retinal Position?}, journal = {Journal of Neuroscience}, volume = {28}, year = {2008}, month = {01/2008}, pages = {10045 - 10055}, abstract = {

Biological visual systems have the remarkable ability to recognize objects despite confounding factors such as object position, size, pose, and lighting. In primates, this ability likely results from neuronal responses at the highest stage of the ventral visual stream [inferior temporal cortex {(IT)]} that signal object identity while tolerating these factors. However, for even the apparently simplest {IT} tolerance (\"invariance\"), tolerance to object position on the retina, little is known about how this feat is achieved. One possibility is that {IT} position tolerance is innate in that discriminatory power for newly learned objects automatically generalizes across position. Alternatively, visual experience plays a role in developing position tolerance. To test these ideas, we trained adult monkeys in a difficult object discrimination task in which their visual experience with novel objects was restricted to a single retinal position. After training, we recorded the spiking activity of an unbiased population of {IT} neurons and found that it contained significantly greater selectivity among the newly learned objects at the experienced position compared with a carefully matched, non-experienced position. Interleaved testing with other objects shows that this difference cannot be attributed to a bias in spatial attention or neuronal sampling. We conclude from these results that, at least under some conditions, full transfer of {IT} neuronal selectivity across retinal position is not automatic. This finding raises the possibility that visual experience plays a role in building neuronal tolerance in the ventral visual stream and the recognition abilities it supports.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.2142-08.2008}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.2142-08.2008}, author = {Cox, D. D. and DiCarlo, J. J.} } @article {82, title = {Discrimination Training Alters Object Representations in Human Extrastriate Cortex}, journal = {Journal of Neuroscience}, volume = {26}, year = {2006}, month = {2006}, pages = {13025 - 13036}, abstract = {

Visual object recognition relies critically on learning. However, little is known about the effect of object learning in human visual cortex, and in particular how the spatial distribution of training effects relates to the distribution of object and face selectivity across the cortex before training. We scanned human subjects with high-resolution functional magnetic resonance imaging {(fMRI)} while they viewed novel object classes, both before and after extensive training to discriminate between exemplars within one of these object classes. Training increased the strength of the response in visual cortex to trained objects compared with untrained objects. However, training did not simply induce a uniform increase in the response to trained objects: the magnitude of this training effect varied substantially across subregions of extrastriate cortex, with some showing a twofold increase in response to trained objects and others (including the right fusiform face area) showing no significant effect of training. Furthermore, the spatial distribution of training effects could not be predicted from the spatial distribution of either pretrained responses or face selectivity. Instead, training changed the spatial distribution of activity across the cortex. These findings support a dynamic view of the ventral visual pathway in which the cortical representation of an object category is continuously modulated by experience.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.2481-06.2006}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.2481-06.2006}, author = {Op de Beeck, H. P. and Baker, C. I. and DiCarlo, J. J. and Kanwisher, N. G.} }