@article {32, title = {Deep Neural Networks Rival the Representation of Primate IT Cortex for Core Visual Object Recognition}, journal = {PLoS Computational Biology}, volume = {10}, year = {2014}, month = {12/2014}, pages = {e1003963}, abstract = {

The primate visual system achieves remarkable visual object recognition performance even in brief presentations, and under changes to object exemplar, geometric transformations, and background variation (a.k.a. core visual object recognition). This remarkable performance is mediated by the representation formed in inferior temporal (IT) cortex. In parallel, recent advances in machine learning have led to ever higher performing models of object recognition using artificial deep neural networks (DNNs). It remains unclear, however, whether the representational performance of DNNs rivals that of the brain. To accurately produce such a comparison, a major difficulty has been a unifying metric that accounts for experimental limitations, such as the amount of noise, the number of neural recording sites, and the number of trials, and computational limitations, such as the complexity of the decoding classifier and the number of classifier training examples. In this work, we perform a direct comparison that corrects for these experimental limitations and computational considerations. As part of our methodology, we propose an extension of \"kernel analysis\" that measures the generalization accuracy as a function of representational complexity. Our evaluations show that, unlike previous bio-inspired models, the latest DNNs rival the representational performance of IT cortex on this visual object recognition task. Furthermore, we show that models that perform well on measures of representational performance also perform well on measures of representational similarity to IT, and on measures of predicting individual IT multi-unit responses. Whether these DNNs rely on computational mechanisms similar to the primate visual system is yet to be determined, but, unlike all previous bio-inspired models, that possibility cannot be ruled out merely on representational performance grounds.

}, doi = {10.1371/journal.pcbi.1003963}, url = {https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003963\&type=printable}, author = {Cadieu, Charles F. and Hong, Ha and Yamins, Daniel L. K. and Pinto, Nicolas and Ardila, Diego and Solomon, Ethan A. and Majaj, Najib J. and DiCarlo, James J.}, editor = {Bethge, Matthias} } @article {36, title = {The Neural Representation Benchmark and its Evaluation on Brain and Machine}, journal = {arXiv}, year = {2013}, month = {01/2013}, type = {preprint}, abstract = {

A key requirement for the development of effective learning representations is their evaluation and comparison to representations we know to be effective. In natural sensory domains, the community has viewed the brain as a source of inspiration and as an implicit benchmark for success. However, it has not been possible to directly test representational learning algorithms directly against the representations contained in neural systems. Here, we propose a new benchmark for visual representations on which we have directly tested the neural representation in multiple visual cortical areas in macaque (utilizing data from [Majaj et al., 2012]), and on which any computer vision algorithm that produces a feature space can be tested. The benchmark measures the effectiveness of the neural or machine representation by computing the classification loss on the ordered eigendecomposition of a kernel matrix [Montavon et al., 2011]. In our analysis we find that the neural representation in visual area IT is superior to visual area V4. In our analysis of representational learning algorithms, we find that three-layer models approach the representational performance of V4 and the algorithm in [Le et al., 2012] surpasses the performance of V4. Impressively, we find that a recent supervised algorithm [Krizhevsky et al., 2012] achieves performance comparable to that of IT for an intermediate level of image variation difficulty, and surpasses IT at a higher difficulty level. We believe this result represents a major milestone: it is the first learning algorithm we have found that exceeds our current estimate of IT representation performance. We hope that this benchmark will assist the community in matching the representational performance of visual cortex and will serve as an initial rallying point for further correspondence between representations derived in brains and machines.

}, doi = {https://arxiv.org/abs/1301.3530}, url = {https://arxiv.org/pdf/1301.3530.pdf}, author = {Cadieu, Charles F. and Ha Hong and Daniel L. K. Yamins and Pinto, Nicolas and Majaj, Najib J and James J. DiCarlo} } @proceedings {43, title = {Comparing-State-of-the-Art Visual Features on Invariant Object Recognition Tasks}, journal = {IEEE Workshop on Applications of Computer Vision (WACV)}, year = {2011}, month = {01/2011}, pages = {463-470}, publisher = {IEEE}, address = {Kona, Hawaii, USA}, abstract = {

Tolerance (\“invariance\”) to identity-preserving image variation (e.g. variation in position, scale, pose, illumination) is a fundamental problem that any visual object recognition system, biological or engineered, must solve. While standard natural image database benchmarks are useful for guiding progress in computer vision, they can fail to probe the ability of a recognition system to solve the invariance problem. Thus, to understand which computational approaches are making progress on solving the invariance problem, we compared and contrasted a variety of state-of-the-art visual representations using synthetic recognition tasks designed to systematically probe invariance. We successfully re-implemented a variety of state-of-the-art visual representations and confirmed their published performance on a natural image benchmark. We here report that most of these representations perform poorly on invariant recognition, but that one representation shows significant performance gains over two baseline representations. We also show how this approach can more deeply illuminate the strengths and weaknesses of different visual representations and thus guide progress on invariant object recognition.

}, issn = {978-1-4244-9497-2}, doi = {10.1109/WACV.2011.5711540}, url = {https://ieeexplore.ieee.org/document/5711540}, author = {Pinto, Nicolas and Barhomi, Y and Cox, David D. and DiCarlo, James J.} } @proceedings {52, title = {A High-Throughput Screening Approach to Biologically-Inspired Object Recognition}, journal = {Learning Workshop-Computation and Systems Neuroscience (COSYNE)}, year = {2010}, month = {03/2010}, address = {Snow Bird, Utah, USA}, author = {Pinto, Nicolas and DiCarlo, James J. and Cox, David D.} } @proceedings {51, title = {Human versus machine: comparing visual object recognition systems on a level playing field}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2010}, month = {02/2010}, address = {Salt Lake City, Utah, USA}, abstract = {

It is received wisdom that biological visual systems easily outmatch current artificial systems at complex visual tasks like object recognition. But have the appropriate comparisons been made? Because artificial systems are improving every day, they may surpass human performance some day. We must understand our progress toward reaching that day, because that success is one of several necessary requirements for \"understanding\" visual object recognition. How large (or small) is the difference in performance between current state-of-the-art object recognition systems and the primate visual system? In practice, the performance comparison of any two object recognition systems requires a focus on the computational crux of the problem and sets of images that engage it. Although it is widely believed that tolerance (\"invariance\") to identity-preserving image variation (e.g. variation in object position, scale, pose, illumination) is critical, systematic comparisons of state-of-the-art artificial visual representations almost always rely on \"natural\" image databases that can fail to probe the ability of a recognition system to solve the invariance problem [Pinto et al PLoS08, COSYNE08, ECCV08, CVPR09]. Thus, to understand how well current state-of-the-art visual representations perform relative to each other, relative to low-level neuronal representations (e.g. retinal-like and V1-like), and relative to high-level representations (e.g. human performance), we tested all of these representations on a common set of visual object recognition tasks that directly engage the invariance problem. Specifically, we used a synthetic testing approach that allows direct engagement of the invariance problem, as well as knowledge and control of all the key parameters that make object recognition challenging. We successfully re-implemented a variety of state-of-the-art visual representations, and we confirmed the high published performance of all of these state-of-the-art representations on large, complex \"natural\â\€ image benchmarks. Surprisingly, we found that most of these representations were weak on our simple synthetic tests of invariant recognition, and only high-level biologically-inspired representations showed performance gains above the neuroscience \"null\" representation (V1-like). While in aggregate, we found that the performance of these state-of-the-art representations pales in comparison to human performance, humans and computers seem to fail in different and potentially enlightening ways when faced with the problem of invariance. We also show how our synthetic testing approach can more deeply illuminate the strengths and weaknesses of different visual representations and thus guide progress on invariant object recognition.

}, doi = {10.3389/conf.fnins.2010.03.00283}, url = {https://www.frontiersin.org/Community/AbstractDetails.aspx?ABS_DOI=10.3389/conf.fnins.2010.03.00283\&eid=770\&sname=Computational_and_Systems_Neuroscience_2010}, author = {Pinto, Nicolas and Majaj, Najib J and Barhomi, Y and Solomon, E. A. and Cox, David D. and DiCarlo, James J.} } @article {56, title = {A High-Throughput Screening Approach to Discovering Good Forms of Biologically Inspired Visual Representation}, journal = {PLoS Computational Biology}, volume = {5}, year = {2009}, month = {2009}, pages = {e1000579}, abstract = {

While many models of biological object recognition share a common set of \"broad-stroke\" properties, the performance of any one model depends strongly on the choice of parameters in a particular instantiation of that model\–e.g., the number of units per layer, the size of pooling kernels, exponents in normalization operations, etc. Since the number of such parameters (explicit or implicit) is typically large and the computational cost of evaluating one particular parameter set is high, the space of possible model instantiations goes largely unexplored. Thus, when a model fails to approach the abilities of biological visual systems, we are left uncertain whether this failure is because we are missing a fundamental idea or because the correct \"parts\" have not been tuned correctly, assembled at sufficient scale, or provided with enough training. Here, we present a high-throughput approach to the exploration of such parameter sets, leveraging recent advances in stream processing hardware (high-end {NVIDIA} graphic cards and the {PlayStation} 3\&$\#$39;s {IBM} Cell Processor). In analogy to high-throughput screening approaches in molecular biology and genetics, we explored thousands of potential network architectures and parameter instantiations, screening those that show promising object recognition performance for further analysis. We show that this approach can yield significant, reproducible gains in performance across an array of basic object recognition tasks, consistently outperforming a variety of state-of-the-art purpose-built vision systems from the literature. As the scale of available computational power continues to expand, we argue that this approach has the potential to greatly accelerate progress in both artificial vision and our understanding of the computational underpinning of biological vision.

}, doi = {10.1371/journal.pcbi.1000579}, url = {https://dx.plos.org/10.1371/journal.pcbi.1000579}, author = {Pinto, Nicolas and Doukhan, David and DiCarlo, James J. and Cox, David D.}, editor = {Friston, Karl J.} } @conference {57, title = {How far can you get with a modern face recognition test set using only simple features?}, booktitle = {IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops (CVPR Workshops)}, year = {2009}, month = {06/2009}, publisher = {IEEE}, organization = {IEEE}, address = {Miami, FL}, abstract = {

In recent years, large databases of natural images have become increasingly popular in the evaluation of face and object recognition algorithms. However, Pinto et al. previously illustrated an inherent danger in using such sets, showing that an extremely basic recognition system, built on a trivial feature set, was able to take advantage of low-level regularities in popular object and face recognition sets, performing on par with many state-of-the-art systems. Recently, several groups have raised the performance \“bar\” for these sets, using more advanced classification tools. However, it is difficult to know whether these improvements are due to progress towards solving the core computational problem, or are due to further improvements in the exploitation of low-level regularities. Here, we show that even modest optimization of the simple model introduced by Pinto et al. using modern multiple kernel learning (MKL) techniques once again yields \“state-of-the-art\” performance levels on a standard face recognition set (\“labeled faces in the wild\”). However, at the same time, even with the inclusion of MKL techniques, systems based on these simple features still fail on a synthetic face recognition test that includes more \“realistic\” view variation by design. These results underscore the importance of building test sets focussed on capturing the central computational challenges of real-world face recognition.

}, isbn = {978-1-4244-3992-8}, doi = {10.1109/CVPR.2009.5206605}, url = {https://ieeexplore.ieee.org/document/5206605}, author = {Pinto, Nicolas and DiCarlo, James J. and Cox, David D.} } @proceedings {60, title = {Unlocking Biologically-Inspired Computer Vision: a High-Throughput Approach}, journal = {NVIDIA GPU Technology Conference}, year = {2009}, month = {2009}, author = {Pinto, Nicolas and Cox, David D. and DiCarlo, James J.} } @proceedings {61, title = {Unlocking Brain-Inspired Computer Vision}, journal = {GPU@BU}, year = {2009}, month = {2009}, author = {Pinto, Nicolas and Cox, David D. and DiCarlo, James J.} } @proceedings {62, title = {The Visual Cortex and GPUs}, journal = {GPU Computing for Biomedical Research}, year = {2009}, month = {2009}, address = {Massachusetts General Hospital, Boston, MA}, author = {Pinto, Nicolas and Cox, David D. and DiCarlo, James J.} } @conference {70, title = {Establishing Good Benchmarks and Baselines for Face Recognition}, booktitle = {European Conference on Computer Vision-Faces in {\textquoteright}Real-Life{\textquoteright} Images Workshop}, year = {2008}, month = {10/2008}, publisher = {EECV}, organization = {EECV}, address = {Marseille, France}, abstract = {

Progress in face recognition relies critically on the creation of test sets against which the performance of various approaches can be evaluated. A good set must capture the essential elements of what makes the problem hard, while conforming to practical scale limitations. However, these goals are often deceptively difficult to achieve. In the related area of object recognition, Pinto et al. [2] demonstrated the potential dangers of using a large, uncontrolled natural image set, showing that an extremely rudimentary vision system (inspired by the early stages of visual processing in the brain) was able to perform on par with many state-of-the-art vision systems on the popular Caltech101 object set [3].\  At the same time, this same rudimentary system was easily defeated by an ostensibly \"simpler\" synthetic recognition test designed to better span the range of real world variation in object pose, position, scale, etc. These results suggested that image sets that look \"natural\" to human observers may nonetheless fail to properly embody the problem of interest, and that care must be taken to establish baselines against which performance can be judged. Here, we repeat this approach for the \"LabeledFaces in the Wild\" (LFW) dataset [1], and for a collection of standard face recognition tests. The goal of the present work is not to compete in the LFW challenge, per se, but to provide a baseline against which the performance of other systems can be judged. In particular, we found that our rudimentary \"baseline\" vision system was able to achieve 68\% correct performance on the LFW challenge, substantially higher than a pure chance\" baseline. We argue that this value might serve as a more useful baseline against which to evaluate absolute performance and argue that the LFW set, while perhaps not perfect, represents an improvement over other standard face sets.

}, author = {Pinto, Nicolas and DiCarlo, James J. and Cox, David D.} } @proceedings {71, title = {A high-throughput screening approach to discovering good forms of visual representation}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2008}, month = {03/2008}, address = {Salt Lake City, Utah, USA}, author = {Cox, David D. and Pinto, Nicolas and Doukhan, David and Corda, B and DiCarlo, James J.} } @proceedings {77, title = {Why is real-world object recognition hard?: Establishing honest benchmarks and baselines for object recognition}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2008}, month = {03/2008}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, author = {Pinto, Nicolas and Cox, David D. and Corda, B and Doukhan, David and DiCarlo, James J.} } @article {65, title = {Why is Real-World Visual Object Recognition Hard?}, journal = {PLoS Computational Biology}, volume = {4}, year = {2008}, month = {01/2008}, pages = {e27}, abstract = {

Progress in understanding the brain mechanisms underlying vision requires the construction of computational models that not only emulate the brain\&$\#$39;s anatomy and physiology, but ultimately match its performance on visual tasks. In recent years, \"natural\" images have become popular in the study of vision and have been used to show apparently impressive progress in building such models. Here, we challenge the use of uncontrolled \"natural\" images in guiding that progress. In particular, we show that a simple V1-like model\–a neuroscientist\&$\#$39;s \"null\" model, which should perform poorly at real-world visual object recognition tasks\–outperforms state-of-the-art object recognition systems (biologically inspired and otherwise) on a standard, ostensibly natural image recognition test. As a counterpoint, we designed a \"simpler\" recognition test to better span the real-world variation in object pose, position, and scale, and we show that this test correctly exposes the inadequacy of the V1-like model. Taken together, these results demonstrate that tests based on uncontrolled natural images can be seriously misleading, potentially guiding progress in the wrong direction. Instead, we reexamine what it means for images to be natural and argue for a renewed focus on the core problem of object recognition\–real-world image variation.

}, doi = {10.1371/journal.pcbi.0040027}, url = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.0040027}, author = {Pinto, Nicolas and Cox, David D and DiCarlo, James J}, editor = {Friston, Karl J} }