@article {29, title = {Comparison of Object Recognition Behavior in Human and Monkey}, journal = {Journal of Neuroscience}, volume = {35}, year = {2015}, month = {02/2015}, pages = {12127 - 12136}, abstract = {

Although the rhesus monkey is used widely as an animal model of human visual processing, it is not known whether invariant visual object recognition behavior is quantitatively comparable across monkeys and humans. To address this question, we systematically compared the core object recognition behavior of two monkeys with that of human subjects. To test true object recognition behavior (rather than image matching), we generated several thousand naturalistic synthetic images of 24 basic-level objects with high variation in viewing parameters and image background. Monkeys were trained to perform binary object recognition tasks on a match-to-sample paradigm. Data from 605 human subjects performing the same tasks on Mechanical Turk were aggregated to characterize \"pooled human\" object recognition behavior, as well as 33 separate Mechanical Turk subjects to characterize individual human subject behavior. Our results show that monkeys learn each new object in a few days, after which they not only match mean human performance but show a pattern of object confusion that is highly correlated with pooled human confusion patterns and is statistically indistinguishable from individual human subjects. Importantly, this shared human and monkey pattern of 3D object confusion is not shared with low-level visual representations (pixels, V1+; models of the retina and primary visual cortex) but is shared with a state-of-the-art computer vision feature representation. Together, these results are consistent with the hypothesis that rhesus monkeys and humans share a common neural shape representation that directly supports object perception.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0573-15.2015}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.0573-15.2015}, author = {Rajalingham, R. and Schmidt, K. and DiCarlo, J. J.} } @article {28, title = {Simple Learned Weighted Sums of Inferior Temporal Neuronal Firing Rates Accurately Predict Human Core Object Recognition Performance}, journal = {Journal of Neuroscience}, volume = {35}, year = {2015}, month = {06/2017}, pages = {13402 - 13418}, abstract = {

To go beyond qualitative models of the biological substrate of object recognition, we ask: can a single ventral stream neuronal linking hypothesis quantitatively account for core object recognition performance over a broad range of tasks? We measured human performance in 64 object recognition tests using thousands of challenging images that explore shape similarity and identity preserving object variation. We then used multielectrode arrays to measure neuronal population responses to those same images in visual areas V4 and inferior temporal (IT) cortex of monkeys and simulated V1 population responses. We tested leading candidate linking hypotheses and control hypotheses, each postulating how ventral stream neuronal responses underlie object recognition behavior. Specifically, for each hypothesis, we computed the predicted performance on the 64 tests and compared it with the measured pattern of human performance. All tested hypotheses based on low- and mid-level visually evoked activity (pixels, V1, and V4) were very poor predictors of the human behavioral pattern. However, simple learned weighted sums of distributed average IT firing rates exactly predicted the behavioral pattern. More elaborate linking hypotheses relying on IT trial-by-trial correlational structure, finer IT temporal codes, or ones that strictly respect the known spatial substructures of IT (\"face patches\") did not improve predictive power. Although these results do not reject those more elaborate hypotheses, they suggest a simple, sufficient quantitative model: each object recognition task is learned from the spatially distributed mean firing rates (100 ms) of \∼60,000 IT neurons and is executed as a simple weighted sum of those firing rates.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.5181-14.2015}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.5181-14.2015}, author = {Majaj, N. J. and Hong, H. and Solomon, E. A. and DiCarlo, J. J.} } @article {31, title = {Performance-optimized hierarchical models predict neural responses in higher visual cortex}, journal = {Proceedings of the National Academy of Sciences}, year = {2014}, month = {10/2014}, pages = {8619 - 8624}, abstract = {

The ventral visual stream underlies key human visual object recognition abilities. However, neural encoding in the higher areas of the ventral stream remains poorly understood. Here, we describe a modeling approach that yields a quantitatively accurate model of inferior temporal (IT) cortex, the highest ventral cortical area. Using high-throughput computational techniques, we discovered that, within a class of biologically plausible hierarchical neural network models, there is a strong correlation between a model\’s categorization performance and its ability to predict individual IT neural unit response data. To pursue this idea, we then identified a high-performing neural network that matches human performance on a range of recognition tasks. Critically, even though we did not constrain this model to match neural data, its top output layer turns out to be highly predictive of IT spiking responses to complex naturalistic images at both the single site and population levels. Moreover, the model\’s intermediate layers are highly predictive of neural responses in the V4 cortex, a midlevel visual area that provides the dominant cortical input to IT. These results show that performance optimization\—applied in a biologically appropriate model class\—can be used to build quantitative predictive models of neural processing.

}, issn = {0027-8424}, doi = {10.1073/pnas.1403112111}, url = {http://www.pnas.org/cgi/doi/10.1073/pnas.1403112111}, author = {Yamins, D. L. K. and Hong, H. and Cadieu, C. F. and Solomon, E. A. and Seibert, D. and DiCarlo, J. J.} } @article {34, title = {Large-Scale, High-Resolution Neurophysiological Maps Underlying fMRI of Macaque Temporal Lobe}, journal = {Journal of Neuroscience}, volume = {33}, year = {2013}, month = {2013}, pages = {15207 - 15219}, abstract = {

Maps obtained by functional magnetic resonance imaging (fMRI) are thought to reflect the underlying spatial layout of neural activity. However, previous studies have not been able to directly compare fMRI maps to high-resolution neurophysiological maps, particularly in higher level visual areas. Here, we used a novel stereo microfocal x-ray system to localize thousands of neural recordings across monkey inferior temporal cortex (IT), construct large-scale maps of neuronal object selectivity at subvoxel resolution, and compare those neurophysiology maps with fMRI maps from the same subjects. While neurophysiology maps contained reliable structure at the sub-millimeter scale, fMRI maps of object selectivity contained information at larger scales (\>2.5 mm) and were only partly correlated with raw neurophysiology maps collected in the same subjects. However, spatial smoothing of neurophysiology maps more than doubled that correlation, while a variety of alternative transforms led to no significant improvement. Furthermore, raw spiking signals, once spatially smoothed, were as predictive of fMRI maps as local field potential signals. Thus, fMRI of the inferior temporal lobe reflects a spatially low-passed version of neurophysiology signals. These findings strongly validate the widespread use of fMRI for detecting large (\>2.5 mm) neuronal domains of object selectivity but show that a complete understanding of even the most pure domains (e.g., faces vs nonface objects) requires investigation at fine scales that can currently only be obtained with invasive neurophysiological methods.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.1248-13.2013}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.1248-13.2013}, author = {Issa, E. B. and Papanastassiou, A. M. and DiCarlo, J. J.} } @article {40, title = {Balanced Increases in Selectivity and Tolerance Produce Constant Sparseness along the Ventral Visual Stream}, journal = {Journal of Neuroscience}, volume = {32}, year = {2012}, month = {07/2012}, pages = {10170 - 10182}, abstract = {

Although popular accounts suggest that neurons along the ventral visual processing stream become increasingly selective for particular objects, this appears at odds with the fact that inferior temporal cortical (IT) neurons are broadly tuned. To explore this apparent contradiction, we compared processing in two ventral stream stages (visual cortical areas V4 and IT) in the rhesus macaque monkey. We confirmed that IT neurons are indeed more selective for conjunctions of visual features than V4 neurons and that this increase in feature conjunction selectivity is accompanied by an increase in tolerance (\"invariance\") to identity-preserving transformations (e.g., shifting, scaling) of those features. We report here that V4 and IT neurons are, on average, tightly matched in their tuning breadth for natural images (\"sparseness\") and that the average V4 or IT neuron will produce a robust firing rate response (\>50\% of its peak observed firing rate) to \∼10\% of all natural images. We also observed that sparseness was positively correlated with conjunction selectivity and negatively correlated with tolerance within both V4 and IT, consistent with selectivity-building and invariance-building computations that offset one another to produce sparseness. Our results imply that the conjunction-selectivity-building and invariance-building computations necessary to support object recognition are implemented in a balanced manner to maintain sparseness at each stage of processing.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.6125-11.2012}, url = {http://www.jneurosci.org/content/32/30/10170.full.pdf+html}, author = {Rust, N. C. and DiCarlo, J. J.} } @article {39, title = {Neuronal Learning of Invariant Object Representation in the Ventral Visual Stream Is Not Dependent on Reward}, journal = {Journal of Neuroscience}, volume = {32}, year = {2012}, month = {09/2012}, pages = {6611 - 6620}, abstract = {

Neurons at the top of primate ventral visual stream [inferior temporal cortex (IT)] have selectivity for objects that is highly tolerant to variation in the object\&$\#$39;s appearance on the retina. Previous nonhuman primate (Macaca mulatta) studies suggest that this neuronal tolerance is at least partly supported by the natural temporal contiguity of visual experience, because altering that temporal contiguity can robustly alter adult IT position and size tolerance. According to that work, it is the statistics of the subject\&$\#$39;s visual experience, not the subject\&$\#$39;s reward, that instruct the specific images that IT treats as equivalent. But is reward necessary for gating this type of learning in the ventral stream? Here we show that this is not the case\—temporal tolerance learning proceeds at the same rate, regardless of reward magnitude and regardless of the temporal co-occurrence of reward, even in a behavioral task that does not require the subject to engage the object images. This suggests that the ventral visual stream uses autonomous, fully unsupervised mechanisms to constantly leverage all visual experience to help build its invariant object representation.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.3786-11.2012}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.3786-11.2012}, author = {Li, N. and DiCarlo, J. J.} } @article {38, title = {Precedence of the Eye Region in Neural Processing of Faces}, journal = {Journal of Neuroscience}, volume = {32}, year = {2012}, month = {09/2013}, pages = {16666 - 16682}, abstract = {

Functional magnetic resonance imaging (fMRI) has revealed multiple subregions in monkey inferior temporal cortex (IT) that are selective for images of faces over other objects. The earliest of these subregions, the posterior lateral face patch (PL), has not been studied previously at the neurophysiological level. Perhaps not surprisingly, we found that PL contains a high concentration of \"face-selective\" cells when tested with standard image sets comparable to those used previously to define the region at the level of fMRI. However, we here report that several different image sets and analytical approaches converge to show that nearly all face-selective PL cells are driven by the presence of a single eye in the context of a face outline. Most strikingly, images containing only an eye, even when incorrectly positioned in an outline, drove neurons nearly as well as full-face images, and face images lacking only this feature led to longer latency responses. Thus, bottom-up face processing is relatively local and linearly integrates features-consistent with parts-based models-grounding investigation of how the presence of a face is first inferred in the IT face processing hierarchy.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.2391-12.2012}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.2391-12.2012}, author = {Issa, E. B. and DiCarlo, J. J.} } @article {45, title = {Selectivity and Tolerance ("Invariance") Both Increase as Visual Information Propagates from Cortical Area V4 to IT}, journal = {Journal of Neuroscience}, volume = {30}, year = {2010}, month = {09/2010}, pages = {12978 - 12995}, abstract = {

Our ability to recognize objects despite large changes in position, size, and context is achieved through computations that are thought to increase both the shape selectivity and the tolerance (\"invariance\") of the visual representation at successive stages of the ventral pathway [visual cortical areas V1, V2, and V4 and inferior temporal cortex {(IT)].} However, these ideas have proven difficult to test. Here, we consider how well population activity patterns at two stages of the ventral stream {(V4} and {IT)} discriminate between, and generalize across, different images. We found that both V4 and {IT} encode natural images with similar fidelity, whereas the {IT} population is much more sensitive to controlled, statistical scrambling of those images. Scrambling sensitivity was proportional to receptive field {(RF)} size in both V4 and {IT,} suggesting that, on average, the number of visual feature conjunctions implemented by a V4 or {IT} neuron is directly related to its {RF} size. We also found that the {IT} population could better discriminate between objects across changes in position, scale, and context, thus directly demonstrating a {V4-to-IT} gain in tolerance. This tolerance gain could be accounted for by both a decrease in single-unit sensitivity to identity-preserving transformations (e.g., an increase in {RF} size) and an increase in the maintenance of rank-order object selectivity within the {RF.} These results demonstrate that, as visual information travels from V4 to {IT,} the population representation is reformatted to become more selective for feature conjunctions and more tolerant to identity preserving transformations, and they reveal the single-unit response properties that underlie that reformatting.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0179-10.2010}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.0179-10.2010}, author = {Rust, N. C. and DiCarlo, J. J.} } @article {55, title = {Balanced increases in selectivity and invariance produce constant sparseness across the ventral visual pathway}, journal = {Journal of Vision}, volume = {9}, year = {2009}, month = {Jan-08-2009}, pages = {738 - 738}, abstract = {

While several studies report neurons in inferotemporal cortex (IT) that are highly selective for particular objects or images, other studies report that neurons in IT tend to be broadly tuned. To investigate how selectivity changes across the ventral visual pathway, we compared the responses of neurons in a mid-level visual area (V4) and a high-level visual area (IT). We first assessed the selectivity of neurons in each area by determining how well each population could discriminate between natural images and \“scrambled\” versions of those images that have the same low-level structure but configured randomly. We found that the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled images. These results suggest that IT neurons are in fact more selective than V4 neurons in terms of the image features that drive these cells. As a second estimate of selectivity, we measured the tuning bandwidth of neurons for natural images (\“sparseness\”). Surprisingly, we found that distributions of sparseness values were indistinguishable between V4 and IT. How can the selectivity for natural image features increase while the tuning bandwidth for natural images remains constant? One possibility is that increases in selectivity for particular image features are offset by increases in tolerance for the (e.g.) position and scale of those features. We found that indeed, measures of tolerance were higher in IT than V4. These results confirm that neurons increase both their selectivity for image features and their tolerance to changes in the position and scale of those features as signals propagate through the ventral pathway. Remarkably, the rates of increase of these two parameters appear to be set such that the tuning bandwidth for natural images is maintained across each stage of cortical processing.

}, doi = {10.1167/9.8.738}, url = {http://jov.arvojournals.org/Article.aspx?doi=10.1167/9.8.738}, author = {Rust, N. C. and DiCarlo, J. J.} } @article {58, title = {A rodent model for the study of invariant visual object recognition}, journal = {Proceedings of the National Academy of Sciences}, volume = {106}, year = {2009}, month = {05/2009}, pages = {8748 - 8753}, abstract = {

The human visual system is able to recognize objects despite tremendous variation in their appearance on the retina resulting from variation in view, size, lighting, etc. This ability\–known as \"invariant\" object recognition\–is central to visual perception, yet its computational underpinnings are poorly understood. Traditionally, nonhuman primates have been the animal model-of-choice for investigating the neuronal substrates of invariant recognition, because their visual systems closely mirror our own. Meanwhile, simpler and more accessible animal models such as rodents have been largely overlooked as possible models of higher-level visual functions, because their brains are often assumed to lack advanced visual processing machinery. As a result, little is known about rodents\&$\#$39; ability to process complex visual stimuli in the face of real-world image variation. In the present work, we show that rats possess more advanced visual abilities than previously appreciated. Specifically, we trained pigmented rats to perform a visual task that required them to recognize objects despite substantial variation in their appearance, due to changes in size, view, and lighting. Critically, rats were able to spontaneously generalize to previously unseen transformations of learned objects. These results provide the first systematic evidence for invariant object recognition in rats and argue for an increased focus on rodents as models for studying high-level visual processing.

}, issn = {0027-8424}, doi = {10.1073/pnas.0811583106}, url = {http://www.pnas.org/cgi/doi/10.1073/pnas.0811583106}, author = {Zoccolan, D. and Oertelt, N. and DiCarlo, J. J. and Cox, D. D.} } @article {68, title = {Does Learned Shape Selectivity in Inferior Temporal Cortex Automatically Generalize Across Retinal Position?}, journal = {Journal of Neuroscience}, volume = {28}, year = {2008}, month = {01/2008}, pages = {10045 - 10055}, abstract = {

Biological visual systems have the remarkable ability to recognize objects despite confounding factors such as object position, size, pose, and lighting. In primates, this ability likely results from neuronal responses at the highest stage of the ventral visual stream [inferior temporal cortex {(IT)]} that signal object identity while tolerating these factors. However, for even the apparently simplest {IT} tolerance (\"invariance\"), tolerance to object position on the retina, little is known about how this feat is achieved. One possibility is that {IT} position tolerance is innate in that discriminatory power for newly learned objects automatically generalizes across position. Alternatively, visual experience plays a role in developing position tolerance. To test these ideas, we trained adult monkeys in a difficult object discrimination task in which their visual experience with novel objects was restricted to a single retinal position. After training, we recorded the spiking activity of an unbiased population of {IT} neurons and found that it contained significantly greater selectivity among the newly learned objects at the experienced position compared with a carefully matched, non-experienced position. Interleaved testing with other objects shows that this difference cannot be attributed to a bias in spatial attention or neuronal sampling. We conclude from these results that, at least under some conditions, full transfer of {IT} neuronal selectivity across retinal position is not automatic. This finding raises the possibility that visual experience plays a role in building neuronal tolerance in the ventral visual stream and the recognition abilities it supports.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.2142-08.2008}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.2142-08.2008}, author = {Cox, D. D. and DiCarlo, J. J.} } @article {54, title = {Fine-Scale Spatial Organization of Face and Object Selectivity in the Temporal Lobe: Do Functional Magnetic Resonance Imaging, Optical Imaging, and Electrophysiology Agree?}, journal = {Journal of Neuroscience}, volume = {28}, year = {2008}, month = {12/2008}, pages = {11796 - 11801}, abstract = {

The spatial organization of the brain\&$\#$39;s object and face representations in the temporal lobe is critical for understanding high-level vision and cognition but is poorly understood. Recently, exciting progress has been made using advanced imaging and physiology methods in humans and nonhuman primates, and the combination of such methods may be particularly powerful. Studies applying these methods help us to understand how neuronal activity, optical imaging, and functional magnetic resonance imaging signals are related within the temporal lobe, and to uncover the fine-grained and large-scale spatial organization of object and face representations in the primate brain.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.3799-08.2008}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.3799-08.2008}, author = {Op de Beeck, H. P. and DiCarlo, J. J. and Goense, J. B. M. and Grill-Spector, K. and Papanastassiou, A. and Tanifuji, M. and Tsao, D. Y.} } @article {78, title = {Trade-Off between Object Selectivity and Tolerance in Monkey Inferotemporal Cortex}, journal = {Journal of Neuroscience}, volume = {27}, year = {2007}, month = {07/2007}, pages = {12292 - 12307}, abstract = {

Object recognition requires both selectivity among different objects and tolerance to vastly different retinal images of the same object, resulting from natural variation in (e.g.) position, size, illumination, and clutter. Thus, discovering neuronal responses that have object selectivity and tolerance to identity-preserving transformations is fundamental to understanding object recognition. Although selectivity and tolerance are found at the highest level of the primate ventral visual stream [the inferotemporal cortex {(IT)],} both properties are highly varied and poorly understood. If an {IT} neuron has very sharp selectivity for a unique combination of object features (\"diagnostic features\"), this might automatically endow it with high tolerance. However, this relationship cannot be taken as given; although some {IT} neurons are highly object selective and some are highly tolerant, the empirical connection of these key properties is unknown. In this study, we systematically measured both object selectivity and tolerance to different identity-preserving image transformations in the spiking responses of a population of monkey {IT} neurons. We found that {IT} neurons with high object selectivity typically have low tolerance (and vice versa), regardless of how object selectivity was quantified and the type of tolerance examined. The discovery of this trade-off illuminates object selectivity and tolerance in {IT} and unifies a range of previous, seemingly disparate results. This finding also argues against the idea that diagnostic conjunctions of features guarantee tolerance. Instead, it is naturally explained by object recognition models in which object selectivity is built through {AND-like} tuning mechanisms.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.1897-07.2007}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.1897-07.2007}, author = {Zoccolan, D. and Kouh, M. and Poggio, T. and DiCarlo, J. J.} } @article {82, title = {Discrimination Training Alters Object Representations in Human Extrastriate Cortex}, journal = {Journal of Neuroscience}, volume = {26}, year = {2006}, month = {2006}, pages = {13025 - 13036}, abstract = {

Visual object recognition relies critically on learning. However, little is known about the effect of object learning in human visual cortex, and in particular how the spatial distribution of training effects relates to the distribution of object and face selectivity across the cortex before training. We scanned human subjects with high-resolution functional magnetic resonance imaging {(fMRI)} while they viewed novel object classes, both before and after extensive training to discriminate between exemplars within one of these object classes. Training increased the strength of the response in visual cortex to trained objects compared with untrained objects. However, training did not simply induce a uniform increase in the response to trained objects: the magnitude of this training effect varied substantially across subregions of extrastriate cortex, with some showing a twofold increase in response to trained objects and others (including the right fusiform face area) showing no significant effect of training. Furthermore, the spatial distribution of training effects could not be predicted from the spatial distribution of either pretrained responses or face selectivity. Instead, training changed the spatial distribution of activity across the cortex. These findings support a dynamic view of the ventral visual pathway in which the cortical representation of an object category is continuously modulated by experience.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.2481-06.2006}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.2481-06.2006}, author = {Op de Beeck, H. P. and Baker, C. I. and DiCarlo, J. J. and Kanwisher, N. G.} }