@article {172, title = {An empirical assay of view-invariant object learning in humans and comparison with baseline image-computable models}, journal = {bioRxiv}, year = {2023}, month = {2023/01/01}, pages = {2022.12.31.522402}, type = {preprint}, abstract = {

How humans learn new visual objects is a longstanding scientific problem. Previous work has led to a diverse collection of models for how it is accomplished, but a current limitation in the field is a lack of empirical benchmarks which can be used to evaluate and compare specific models against each other. Here, we use online psychophysics to measure human behavioral learning trajectories over a set of tasks involving novel 3D objects. Consistent with intuition, these results show that humans generally require very few images (\≈ 6) to approach their asymptotic accuracy, find some object discriminations more easy to learn than others, and generalize quite well over a range of image transformations after even one view of each object. We then use those data to develop benchmarks that may be used to evaluate a learning model\’s similarity to humans. We make these data and benchmarks publicly available [GitHub], and, to our knowledge, they are currently the largest publicly-available collection of learning-related psychophysics data in humans. Additionally, to serve as baselines for those benchmarks, we implement and test a large number of baseline models (n=1,932), each based on a standard cognitive theory of learning: that humans re-represent images in a fixed, Euclidean space, then learn linear decision boundaries in that space to identify objects in future images. We find some of these baseline models make surprisingly accurate predictions. However, we also find reliable prediction gaps between all baseline models and humans, particularly in the few-shot learning setting.Competing Interest StatementThe authors have declared no competing interest.

}, doi = {https://www.biorxiv.org/content/10.1101/2022.12.31.522402v1}, url = {https://www.biorxiv.org/content/biorxiv/early/2023/01/02/2022.12.31.522402.full.pdf}, author = {Lee, Michael J. and DiCarlo, James J.} } @article {4, title = {Evidence that recurrent circuits are critical to the ventral stream{\textquoteright}s execution of core object recognition behavior}, journal = {Nature Neuroscience}, volume = {22}, year = {2019}, month = {01/2019}, pages = {974 - 983}, issn = {1097-6256}, doi = {10.1038/s41593-019-0392-5}, url = {http://www.nature.com/articles/s41593-019-0392-5}, author = {Kar, Kohitij and Kubilius, Jonas and Schmidt, Kailyn and Issa, Elias B. and DiCarlo, James J.} } @article {19, title = {Evidence that recurrent circuits are critical to the ventral stream{\textquoteright}s execution of core object recognition behavior}, journal = {bioRxiv}, year = {2018}, month = {06/2018}, type = {preprint}, abstract = {

Non-recurrent deep convolutional neural networks (DCNNs) are currently the best models of core object recognition; a behavior supported by the densely recurrent primate ventral stream, culminating in the inferior temporal (IT) cortex. Are these recurrent circuits critical to ventral stream\&$\#$39;s execution of this behavior? We reasoned that, if recurrence is critical, then primates should outperform feedforward-only DCNNs for some images, and that these images should require additional processing time beyond the feedforward IT response. Here we first used behavioral methods to discover hundreds of these \"challenge\" images. Second, using large-scale IT electrophysiology in animals performing core recognition tasks, we observed that behaviorally-sufficient, linearly-decodable object identity solutions emerged ~30ms (on average) later in IT for challenge images compared to DCNN and primate performance-matched \"control\" images. We observed these same late solutions even during passive viewing. Third, consistent with a failure of feedforward computations, the behaviorally-critical late-phase IT population response patterns evoked by the challenge images were poorly predicted by DCNN activations. Interestingly, deeper CNNs better predicted these late IT responses, suggesting a functional equivalence between recurrence and additional nonlinear transformations. Our results argue that automatically-evoked recurrent circuits are critical even for rapid object identification. By precisely comparing current DCNNs, primate behavior and IT population dynamics, we provide guidance for future recurrent model development.

}, doi = {https://doi.org/10.1101/354753}, url = {https://www.biorxiv.org/content/10.1101/354753v1.full.pdf}, author = {Kar, Kohitij and Kubilius, Jonas and Schmidt, Kailyn and Issa, Elias B and DiCarlo, James J.} } @article {25, title = {Eight open questions in the computational modeling of higher sensory cortex}, journal = {Current Opinion in Neurobiology}, volume = {37}, year = {2016}, month = {01/2016}, pages = {114 - 120}, abstract = {

Propelled by advances in biologically inspired computer vision and artificial intelligence, the past five years have seen significant progress in using deep neural networks to model response patterns of neurons in visual cortex. In this paper, we briefly review this progress and then discuss eight key \&$\#$39;open questions\&$\#$39; that we believe will drive research in computational models of sensory systems over the next five years, both in visual cortex and beyond.

}, issn = {1873-6882}, doi = {10.1016/j.conb.2016.02.001}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0959438816300022}, author = {Yamins, Daniel LK and DiCarlo, James J} } @article {27, title = {Explicit information for category-orthogonal object properties increases along the ventral stream}, journal = {Nature Neuroscience}, volume = {19}, year = {2016}, month = {02/2016}, pages = {613 - 622}, abstract = {

Extensive research has revealed that the ventral visual stream hierarchically builds a robust representation for supporting visual object categorization tasks. We systematically explored the ability of multiple ventral visual areas to support a variety of \&$\#$39;category-orthogonal\&$\#$39; object properties such as position, size and pose. For complex naturalistic stimuli, we found that the inferior temporal (IT) population encodes all measured category-orthogonal object properties, including those properties often considered to be low-level features (for example, position), more explicitly than earlier ventral stream areas. We also found that the IT population better predicts human performance patterns across properties. A hierarchical neural network model based on simple computational principles generates these same cross-area patterns of information. Taken together, our empirical results support the hypothesis that all behaviorally relevant object properties are extracted in concert up the ventral visual hierarchy, and our computational model explains how that hierarchy might be built.

}, issn = {1097-6256}, doi = {10.1038/nn.4247}, url = {http://www.nature.com/articles/nn.4247}, author = {Hong, Ha and Yamins, Daniel L K and Majaj, Najib J and DiCarlo, James J} } @conference {70, title = {Establishing Good Benchmarks and Baselines for Face Recognition}, booktitle = {European Conference on Computer Vision-Faces in {\textquoteright}Real-Life{\textquoteright} Images Workshop}, year = {2008}, month = {10/2008}, publisher = {EECV}, organization = {EECV}, address = {Marseille, France}, abstract = {

Progress in face recognition relies critically on the creation of test sets against which the performance of various approaches can be evaluated. A good set must capture the essential elements of what makes the problem hard, while conforming to practical scale limitations. However, these goals are often deceptively difficult to achieve. In the related area of object recognition, Pinto et al. [2] demonstrated the potential dangers of using a large, uncontrolled natural image set, showing that an extremely rudimentary vision system (inspired by the early stages of visual processing in the brain) was able to perform on par with many state-of-the-art vision systems on the popular Caltech101 object set [3].\  At the same time, this same rudimentary system was easily defeated by an ostensibly \"simpler\" synthetic recognition test designed to better span the range of real world variation in object pose, position, scale, etc. These results suggested that image sets that look \"natural\" to human observers may nonetheless fail to properly embody the problem of interest, and that care must be taken to establish baselines against which performance can be judged. Here, we repeat this approach for the \"LabeledFaces in the Wild\" (LFW) dataset [1], and for a collection of standard face recognition tests. The goal of the present work is not to compete in the LFW challenge, per se, but to provide a baseline against which the performance of other systems can be judged. In particular, we found that our rudimentary \"baseline\" vision system was able to achieve 68\% correct performance on the LFW challenge, substantially higher than a pure chance\" baseline. We argue that this value might serve as a more useful baseline against which to evaluate absolute performance and argue that the LFW set, while perhaps not perfect, represents an improvement over other standard face sets.

}, author = {Pinto, Nicolas and DiCarlo, James J. and Cox, David D.} } @proceedings {95, title = {The effect of visual experience on the position tolerance of primate object representations}, journal = {Society for Neuroscience}, year = {2004}, month = {10/2004}, publisher = {SFN}, address = {San Diego, CA, USA}, author = {Cox, David D. and DiCarlo, James J.} }