@article {174, title = {Catalyzing next-generation Artificial Intelligence through NeuroAI}, journal = {Nature Communications}, volume = {14}, year = {2023}, month = {Jan-12-2023}, pages = {1597}, abstract = {

Neuroscience has long been an essential driver of progress in artificial intelligence (AI). We propose that to accelerate progress in AI, we must invest in fundamental research in NeuroAI. A core component of this is the embodied Turing test, which challenges AI animal models to interact with the sensorimotor world at skill levels akin to their living counterparts. The embodied Turing test shifts the focus from those capabilities like game playing and language that are especially well-developed or uniquely human to those capabilities \– inherited from over 500 million years of evolution \– that are shared with all animals. Building models that can pass the embodied Turing test will provide a roadmap for the next generation of AI.

}, doi = {10.1038/s41467-023-37180-x}, url = {https://www.nature.com/articles/s41467-023-37180-xhttps://www.nature.com/articles/s41467-023-37180-x.pdfhttps://www.nature.com/articles/s41467-023-37180-x.pdfhttps://www.nature.com/articles/s41467-023-37180-x}, author = {Zador, Anthony and Escola, Sean and Richards, Blake and {\"O}lveczky, Bence and Bengio, Yoshua and Boahen, Kwabena and Botvinick, Matthew and Chklovskii, Dmitri and Churchland, Anne and Clopath, Claudia and DiCarlo, James and Ganguli, Surya and Hawkins, Jeff and {\"o}rding, Konrad and Koulakov, Alexei and LeCun, Yann and Lillicrap, Timothy and Marblestone, Adam and Olshausen, Bruno and Pouget, Alexandre and Savin, Cristina and Sejnowski, Terrence and Simoncelli, Eero and Solla, Sara and Sussillo, David and Tolias, Andreas S. and Tsao, Doris} } @article {158, title = {The ThreeDWorld Transport Challenge: A Visually Guided Task-and-Motion Planning Benchmark for Physically Realistic Embodied AI}, journal = {arXiv}, year = {2021}, month = {03/25/2021}, type = {preprint}, abstract = {

We introduce a visually-guided and physics-driven taskand-motion planning benchmark, which we call the ThreeDWorld Transport Challenge. In this challenge, an embodied agent equipped with two 9-DOF articulated arms is spawned randomly in a simulated physical home environment. The agent is required to find a small set of objects scattered around the house, pick them up, and transport them to a desired final location. We also position containers around the house that can be used as tools to assist with transporting objects efficiently. To complete the task, an embodied agent must plan a sequence of actions to change the state of a large number of objects in the face of realistic physical constraints. We build this benchmark challenge using the ThreeDWorld simulation: a virtual 3D environment where all objects respond to physics, and where can be controlled using a fully physics-driven navigation and interaction API. We evaluate several existing agents on this benchmark. Experimental results suggest that: 1) a pure RL model struggles on this challenge; 2) hierarchical planningbased agents can transport some objects but still far from solving this task. We anticipate that this benchmark will empower researchers to develop more intelligent physicsdriven robots for the physical world.\ 

}, doi = {arXiv:2103.14025}, url = {https://arxiv.org/abs/2103.14025}, author = {Gan, Chuang and Zhou, Siyuan and Schwartz, Jeremy and Alter, Seth and Bhandwaldar, Abhishek and Gutfreund, Dan and Yamins, Daniel LK and DiCarlo, James J and McDermott, Josh and Torralba, Antonio} } @article {167, title = {Unsupervised neural network models of the ventral visual stream}, journal = {Proceedings of the National Academy of Sciences}, volume = {118}, year = {2021}, month = {Jul-01-2022}, pages = {e2014196118}, abstract = {

Deep neural networks currently provide the best quantitative models of the response patterns of neurons throughout the primate ventral visual stream. However, such networks have remained implausible as a model of the development of the ventral stream, in part because they are trained with supervised methods requiring many more labels than are accessible to infants during development. Here, we report that recent rapid progress in unsupervised learning has largely closed this gap. We find that neural network models learned with deep unsupervised contrastive embedding methods achieve neural prediction accuracy in multiple ventral visual cortical areas that equals or exceeds that of models derived using today\’s best supervised methods and that the mapping of these neural network models\’ hidden layers is neuroanatomically consistent across the ventral stream. Strikingly, we find that these methods produce brain-like representations even when trained solely with real human child developmental data collected from head-mounted cameras, despite the fact that these datasets are noisy and limited. We also find that semisupervised deep contrastive embeddings can leverage small numbers of labeled examples to produce representations with substantially improved error-pattern consistency to human behavior. Taken together, these results illustrate a use of unsupervised learning to provide a quantitative model of a multiarea cortical brain system and present a strong candidate for a biologically plausible computational theory of primate sensory learning.

}, issn = {0027-8424}, doi = {10.1073/pnas.2014196118}, url = {http://www.pnas.org/lookup/doi/10.1073/pnas.2014196118}, author = {Zhuang, Chengxu and Yan, Siming and Nayebi, Aran and Schrimpf, Martin and Frank, Michael C. and DiCarlo, James J. and Yamins, Daniel L. K.} } @article {150, title = {Unsupervised Neural Network Models of the Ventral Visual Stream}, journal = {bioRxiv}, year = {2020}, month = {June 18, 2020}, type = {preprint}, abstract = {

Deep neural networks currently provide the best quantitative models of the response patterns of neurons throughout the primate ventral visual stream. However, such networks have remained implausible as a model of the development of the ventral stream, in part because they are trained with supervised methods requiring many more labels than are accessible to infants during development. Here, we report that recent rapid progress in unsupervised learning has largely closed this gap. We find that neural network models learned with deep unsupervised contrastive embedding methods achieve neural prediction accuracy in multiple ventral visual cortical areas that equals or exceeds that of models derived using today\’s best supervised methods, and that the mapping of these neural network models\’ hidden layers is neuroanatomically consistent across the ventral stream. Moreover, we find that these methods produce brain-like representations even when trained on noisy and limited data measured from real children\’s developmental experience. We also find that semi-supervised deep contrastive embeddings can leverage small numbers of labelled examples to produce representations with substantially improved error-pattern consistency to human behavior. Taken together, these results suggest that deep contrastive embedding objectives may be a biologically-plausible computational theory of primate visual development.

}, doi = {10.1101/2020.06.16.155556}, url = {https://www.biorxiv.org/content/10.1101/2020.06.16.155556v1.abstract}, author = {Zhuang, Chengxu and Yan, Siming and Nayebi, Aran and Schrimpf, Martin and Frank, Michael and DiCarlo, James J. and Yamins, Daniel L.K.} } @conference {166, title = {Using Brain-Score to Evaluate and Build Neural Networks for Brain-Like Object Recognition}, booktitle = {Computational and Systems Neuroscience (COSYNE)}, year = {2019}, address = {Denver, CO}, author = {Schrimpf, Martin and Kubilius, Jonas and Hong, Ha and Majaj, Najib and Rajalingham, Rishi and Issa, Elias B and Kar, Kohitij and Ziemba, Corey M and Bashivan, Pouya and Prescott-Roy, Jonathan and Schmidt, Kailyn and Yamins, Daniel LK and DiCarlo, James J} } @article {35, title = {Shape Similarity, Better than Semantic Membership, Accounts for the Structure of Visual Object Representations in a Population of Monkey Inferotemporal Neurons}, journal = {PLoS Computational Biology}, volume = {9}, year = {2013}, month = {08/2013}, pages = {e1003167}, abstract = {

The anterior inferotemporal cortex (IT) is the highest stage along the hierarchy of visual areas that, in primates, processes visual objects. Although several lines of evidence suggest that IT primarily represents visual shape information, some recent studies have argued that neuronal ensembles in IT code the semantic membership of visual objects (i.e., represent conceptual classes such as animate and inanimate objects). In this study, we investigated to what extent semantic, rather than purely visual information, is represented in IT by performing a multivariate analysis of IT responses to a set of visual objects. By relying on a variety of machine-learning approaches (including a cutting-edge clustering algorithm that has been recently developed in the domain of statistical physics), we found that, in most instances, IT representation of visual objects is accounted for by their similarity at the level of shape or, more surprisingly, low-level visual properties. Only in a few cases we observed IT representations of semantic classes that were not explainable by the visual similarity of their members. Overall, these findings reassert the primary function of IT as a conveyor of explicit visual shape information, and reveal that low-level visual properties are represented in IT to a greater extent than previously appreciated. In addition, our work demonstrates how combining a variety of state-of-the-art multivariate approaches, and carefully estimating the contribution of shape similarity to the representation of object categories, can substantially advance our understanding of neuronal coding of visual objects in cortex.

}, doi = {10.1371/journal.pcbi.1003167}, url = {https://dx.plos.org/10.1371/journal.pcbi.1003167}, author = {Baldassi, Carlo and Alemi-Neissi, Alireza and Pagan, Marino and DiCarlo, James J. and Zecchina, Riccardo and Zoccolan, Davide} } @article {41, title = {How Does the Brain Solve Visual Object Recognition?}, journal = {Neuron}, volume = {73}, year = {2012}, month = {01/2012}, pages = {415 - 434}, abstract = {

Mounting evidence suggests that \&$\#$39;core object recognition,\&$\#$39; the ability to rapidly recognize objects despite substantial appearance variation, is solved in the brain via a cascade of reflexive, largely feedforward computations that culminate in a powerful neuronal representation in the inferior temporal cortex. However, the algorithm that produces this solution remains poorly understood. Here we review evidence ranging from\ individual neurons and neuronal populations to behavior and computational models. We propose that understanding this algorithm will require using neuronal and psychophysical data to sift through many computational models, each based on building blocks of small, canonical subnetworks with a common functional goal.

}, issn = {08966273}, doi = {10.1016/j.neuron.2012.01.010}, url = {https://www.sciencedirect.com/science/article/pii/S089662731200092X}, author = {DiCarlo, ~J. and Zoccolan, Davide and Rust, ~C.} } @proceedings {44, title = {From luminance to semantics: how natural objects are represented in monkey inferotemporal cortex}, journal = {Computational and Systems Neuroscience (COSYNE)}, year = {2011}, month = {02/2011}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, abstract = {

In primates, visual object information is processed through a hierarchy of cortico-cortical stages that culminates with the inferotemporal cortex (IT). Although the nature of visual processing in IT is still poorly understood, several lines of evidence suggest that IT conveys an explicit object representation that can directly serve as a basis for decision, action and memory - e.g., it can support flexible formation of semantic categories in downstream areas, such as prefrontal and perirhinal cortex. However, some recent studies (Kiani et al, 2007; Kriegeskorte et al, 2008) have argued that IT neuronal ensembles may themselves code the semantic membership of visual objects (i.e., represent such abstract conceptual classes such as animate and inanimate objects, animals, etc). In this study, we have applied an array of multi-variate computational approaches to investigate the nature of visual objects\&$\#$39; representation in IT. Our results show that IT neuronal ensembles represent a surprisingly broad spectrum of visual features complexity, ranging from low-level visual properties (e.g., brightness), to visual patterns of intermediate complexity (e.g., star-like shapes), to complex objects (e.g., four-leg animals) that appear to be coded so invariantly that their clustering in the IT neuronal space is not easily accountable by any similarity metric we used. On the one hand, these findings show that IT supports recognition of low-level properties of the visual input that are typically though to be extracted by lower-level visual areas. On the other hand, IT appears to convey such an explicit representation of some object classes that coding of semantic membership in IT (at least for a few categories) cannot be excluded. Overall, these results shed new light on IT amazing pluripotency in supporting recognition tasks as diverse as detection of brightness and categorization of complex shapes.

}, url = {http://www.cosyne.org/c/index.php?title=Cosyne_11_posters/I-89}, author = {Pagan, Marino and Neissi, AA and Baldassi, Carlo and Zecchina, Riccardo and DiCarlo, James J. and Zoccolan, Davide} } @article {58, title = {A rodent model for the study of invariant visual object recognition}, journal = {Proceedings of the National Academy of Sciences}, volume = {106}, year = {2009}, month = {05/2009}, pages = {8748 - 8753}, abstract = {

The human visual system is able to recognize objects despite tremendous variation in their appearance on the retina resulting from variation in view, size, lighting, etc. This ability\–known as \"invariant\" object recognition\–is central to visual perception, yet its computational underpinnings are poorly understood. Traditionally, nonhuman primates have been the animal model-of-choice for investigating the neuronal substrates of invariant recognition, because their visual systems closely mirror our own. Meanwhile, simpler and more accessible animal models such as rodents have been largely overlooked as possible models of higher-level visual functions, because their brains are often assumed to lack advanced visual processing machinery. As a result, little is known about rodents\&$\#$39; ability to process complex visual stimuli in the face of real-world image variation. In the present work, we show that rats possess more advanced visual abilities than previously appreciated. Specifically, we trained pigmented rats to perform a visual task that required them to recognize objects despite substantial variation in their appearance, due to changes in size, view, and lighting. Critically, rats were able to spontaneously generalize to previously unseen transformations of learned objects. These results provide the first systematic evidence for invariant object recognition in rats and argue for an increased focus on rodents as models for studying high-level visual processing.

}, issn = {0027-8424}, doi = {10.1073/pnas.0811583106}, url = {http://www.pnas.org/cgi/doi/10.1073/pnas.0811583106}, author = {Zoccolan, D. and Oertelt, N. and DiCarlo, J. J. and Cox, D. D.} } @article {53, title = {What Response Properties Do Individual Neurons Need to Underlie Position and Clutter {\textquotedblleft}Invariant{\textquotedblright} Object Recognition?}, journal = {Journal of Neurophysiology}, volume = {102}, year = {2009}, month = {01/2009}, pages = {360 - 376}, abstract = {

Primates can easily identify visual objects over large changes in retinal position\–a property commonly referred to as position \"invariance.\" This ability is widely assumed to depend on neurons in inferior temporal cortex {(IT)} that can respond selectively to isolated visual objects over similarly large ranges of retinal position. However, in the real world, objects rarely appear in isolation, and the interplay between position invariance and the representation of multiple objects (i.e., clutter) remains unresolved. At the heart of this issue is the intuition that the representations of nearby objects can interfere with one another and that the large receptive fields needed for position invariance can exacerbate this problem by increasing the range over which interference acts. Indeed, most {IT} neurons\&$\#$39; responses are strongly affected by the presence of clutter. While external mechanisms (such as attention) are often invoked as a way out of the problem, we show (using recorded neuronal data and simulations) that the intrinsic properties of {IT} population responses, by themselves, can support object recognition in the face of limited clutter. Furthermore, we carried out extensive simulations of hypothetical neuronal populations to identify the essential individual-neuron ingredients of a good population representation. These simulations show that the crucial neuronal property to support recognition in clutter is not preservation of response magnitude, but preservation of each neuron\&$\#$39;s rank-order object preference under identity-preserving image transformations (e.g., clutter). Because {IT} neuronal responses often exhibit that response property, while neurons in earlier visual areas (e.g., V1) do not, we suggest that preserving the rank-order object preference regardless of clutter, rather than the response magnitude, more precisely describes the goal of individual neurons at the top of the ventral visual stream.

}, issn = {0022-3077}, doi = {10.1152/jn.90745.2008}, url = {https://www.physiology.org/doi/10.1152/jn.90745.2008}, author = {Li, Nuo and Cox, David D. and Zoccolan, Davide and DiCarlo, James J.} } @proceedings {73, title = {Is the rodent a valuable model system for studying invariant object recognition?}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2008}, month = {03/2008}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, abstract = {

Despite the many advantages rodents offer in terms of experimental accessibility, they have never been extensively used as models to investigate the neuronal processing of visual objects. A crucial step in establishing whether rodents are suitable models for the study of object vision, is to assess if they are capable of invariant object recognition \– i.e., recognition of visual objects across the range of transformations that they typically undergo during natural vision (e.g., changes in position and size). In this study, we tested rats (Long-Evans) capability to discriminate between two geometrical shapes (square and triangle) presented at different sizes, positions and orientations. In each trial, a single shape was presented on a computer monitor and the animal had to report its identity. The experiment consisted of three phases. In phase I, each animal learned to discriminate between the two shapes presented at fixed size (40\° of visual angle), position (center of the monitor), and orientation. Na\ïve rats typically achieved \> 70\% correct performance in 2-3 weeks of training. In phase II, the animals were trained to perform the task while either the size, the horizontal position, or the orientation of the shapes was separately varied. Rats readily acquired this task (\> 70\% correct performance) across sizes ranging from 40\° to 10\°, positions spanning \±12\°, and orientations spanning \±40\°. During phase III of the experiment, we asked if rats could generalize to novel combinations of size, position, and orientation (a total of 100 transformations of each target shape were tested). Performance was typically \> 70\% correct for nearly all of these previously unseen transformations (performance of one rat over a subset of the tested conditions is shown in the figure) and even for a fraction of transformations for which feedback was withheld. These results show that rats can readily learn to: 1) discriminate between simple visual shapes; 2) disregard variations in their position, size and orientation; and 3) generalize to novel views. This suggests that the rat visual system contains the fundamental mechanisms that support object recognition. Therefore, given the broad ranges of experimental approaches available in rats, they may be a powerful new model system to study the neuronal basis of invariant object recognition

}, url = {http://www.cosyne.org/c/images/8/8e/Cosyne_pf_new.pdf}, author = {Zoccolan, Davide and Cox, David D. and Oertelt, Nadja and Radwam, Basma and Tsang, Sabrina and DiCarlo, James J.} } @article {78, title = {Trade-Off between Object Selectivity and Tolerance in Monkey Inferotemporal Cortex}, journal = {Journal of Neuroscience}, volume = {27}, year = {2007}, month = {07/2007}, pages = {12292 - 12307}, abstract = {

Object recognition requires both selectivity among different objects and tolerance to vastly different retinal images of the same object, resulting from natural variation in (e.g.) position, size, illumination, and clutter. Thus, discovering neuronal responses that have object selectivity and tolerance to identity-preserving transformations is fundamental to understanding object recognition. Although selectivity and tolerance are found at the highest level of the primate ventral visual stream [the inferotemporal cortex {(IT)],} both properties are highly varied and poorly understood. If an {IT} neuron has very sharp selectivity for a unique combination of object features (\"diagnostic features\"), this might automatically endow it with high tolerance. However, this relationship cannot be taken as given; although some {IT} neurons are highly object selective and some are highly tolerant, the empirical connection of these key properties is unknown. In this study, we systematically measured both object selectivity and tolerance to different identity-preserving image transformations in the spiking responses of a population of monkey {IT} neurons. We found that {IT} neurons with high object selectivity typically have low tolerance (and vice versa), regardless of how object selectivity was quantified and the type of tolerance examined. The discovery of this trade-off illuminates object selectivity and tolerance in {IT} and unifies a range of previous, seemingly disparate results. This finding also argues against the idea that diagnostic conjunctions of features guarantee tolerance. Instead, it is naturally explained by object recognition models in which object selectivity is built through {AND-like} tuning mechanisms.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.1897-07.2007}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.1897-07.2007}, author = {Zoccolan, D. and Kouh, M. and Poggio, T. and DiCarlo, J. J.} } @proceedings {85, title = {Flexible and robust object recognition in inferior temporal cortex supported by neurons with limited position and clutter tolerance}, journal = {Society for Neuroscience}, year = {2006}, month = {10/2006}, publisher = {SFN}, address = {Atlanta, GA, USA}, author = {Li, N. and Cox, David D. and Zoccolan, Davide and DiCarlo, James J.} } @proceedings {86, title = {Trade-off between shape selectivity and tolerance to identity-preserving transformations in monkey inferotemporal cortex}, journal = {Gordon Conference: Sensation and the Natural Environment}, year = {2006}, month = {2006}, address = {Bozeman, MT, USA}, author = {Zoccolan, Davide and Kouh, M and Poggio, Tomaso and DiCarlo, James J.} } @article {87, title = {Multiple Object Response Normalization in Monkey Inferotemporal Cortex}, journal = {Journal of Neuroscience}, volume = {25}, year = {2005}, month = {07/2005}, pages = {8150 - 8164}, abstract = {

The highest stages of the visual ventral pathway are commonly assumed to provide robust representation of object identity by disregarding confounding factors such as object position, size, illumination, and the presence of other objects (clutter). However, whereas neuronal responses in monkey inferotemporal cortex (IT) can show robust tolerance to position and size changes, previous work shows that responses to preferred objects are usually reduced by the presence of nonpreferred objects. More broadly, we do not yet understand multiple object representation in IT. In this study, we systematically examined IT responses to pairs and triplets of objects in three passively viewing monkeys across a broad range of object effectiveness. We found that, at least under these limited clutter conditions, a large fraction of the response of each IT neuron to multiple objects is reliably predicted as the average of its responses to the constituent objects in isolation. That is, multiple object responses depend primarily on the relative effectiveness of the constituent objects, regardless of object identity. This average effect becomes virtually perfect when populations of IT neurons are pooled. Furthermore, the average effect cannot simply be explained by attentional shifts but behaves as a primarily feedforward response property. Together, our observations are most consistent with mechanistic models in which IT neuronal outputs are normalized by summed synaptic drive into IT or spiking activity within IT and suggest that normalization mechanisms previously revealed at earlier visual areas are operating throughout the ventral visual stream.

}, keywords = {Animals, Brain Mapping, Macaca mulatta, Male, Photic Stimulation, Posture, Psychology, Recognition, Temporal Lobe, Visual Pathways, Visual Perception}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.2058-05.2005}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.2058-05.2005}, author = {Zoccolan, D. and Cox, David D. and DiCarlo, James J.} } @proceedings {92, title = {Multiple object response normalization in monkey inferotemporal cortex}, journal = {Society for Neuroscience}, year = {2005}, month = {11/2005}, publisher = {SFN}, address = {Washington, DC, USA}, author = {Zoccolan, Davide and Cox, David D. and DiCarlo, James J.} }