@article {174, title = {Catalyzing next-generation Artificial Intelligence through NeuroAI}, journal = {Nature Communications}, volume = {14}, year = {2023}, month = {Jan-12-2023}, pages = {1597}, abstract = {

Neuroscience has long been an essential driver of progress in artificial intelligence (AI). We propose that to accelerate progress in AI, we must invest in fundamental research in NeuroAI. A core component of this is the embodied Turing test, which challenges AI animal models to interact with the sensorimotor world at skill levels akin to their living counterparts. The embodied Turing test shifts the focus from those capabilities like game playing and language that are especially well-developed or uniquely human to those capabilities \– inherited from over 500 million years of evolution \– that are shared with all animals. Building models that can pass the embodied Turing test will provide a roadmap for the next generation of AI.

}, doi = {10.1038/s41467-023-37180-x}, url = {https://www.nature.com/articles/s41467-023-37180-xhttps://www.nature.com/articles/s41467-023-37180-x.pdfhttps://www.nature.com/articles/s41467-023-37180-x.pdfhttps://www.nature.com/articles/s41467-023-37180-x}, author = {Zador, Anthony and Escola, Sean and Richards, Blake and {\"O}lveczky, Bence and Bengio, Yoshua and Boahen, Kwabena and Botvinick, Matthew and Chklovskii, Dmitri and Churchland, Anne and Clopath, Claudia and DiCarlo, James and Ganguli, Surya and Hawkins, Jeff and {\"o}rding, Konrad and Koulakov, Alexei and LeCun, Yann and Lillicrap, Timothy and Marblestone, Adam and Olshausen, Bruno and Pouget, Alexandre and Savin, Cristina and Sejnowski, Terrence and Simoncelli, Eero and Solla, Sara and Sussillo, David and Tolias, Andreas S. and Tsao, Doris} } @article {178, title = {fROI-level computational models enable broad-scale experimental testing and expose key divergences between models and brains}, journal = {Journal of Vision}, volume = {23}, year = {2023}, month = {2023}, pages = {5788 - 5788}, abstract = {

Deep convolutional neural network (DNN)- based models have emerged as our leading hypotheses of human vision. Here we describe, and expand upon, our latest effort to use DNN models of brain regions to explain key results from previous cognitive neuroscience and psychology experiments. Many stimuli in these prior experiments were highly manipulated (e.g. scrambled body parts, face parts, re-arranged spatial positions) often outside the domain of natural stimuli. These results can therefore be considered as tests of model generalization beyond naturalistic stimuli. We first performed these tests on the fusiform face area (FFA), parahippocampal place area (PPA) and the extrastriate body area (EBA). Our previous results (presented in VSS2022) showed that our fROI-level models recapitulate several key results from prior studies. We also observed that models did not perform as well on non-naturalistic stimuli. Here we extend our model evaluation metrics in two ways. First, we replicated findings from the original paper on the EBA (Downing et al 2001) that the EBA responds as highly to line drawings of bodies, and symbolic stick figures as to natural images of bodies (and not to control conditions like faces and objects). Second, we find that none of the computational models explain this pattern of observed responses, though models trained with language-based supervision (like CLIP) do better than other models. Together, our results on symbolic body images expose the bounds of current computational models. This progress was made possible only because of fROI-level modeling procedures, and opens up new ways to understand the power and limitations of current models and test novel hypotheses completely in-silico.

}, isbn = {1534-7362}, url = {https://doi.org/10.1167/jov.23.9.5788}, author = {Mieczkowski, Elizabeth and Abate, Alex and De Faria, Willian and Lydic, Kirsten and DiCarlo, James and Kanwisher, Nancy and Apurva Ratan Murty, N.} } @article {177, title = {A Unifying Principle for the Functional Organization of Visual Cortex}, journal = {bioRxiv}, year = {2023}, month = {2023}, abstract = {

A key feature of many cortical systems is functional organization: the arrangement of neurons with specific functional properties in characteristic spatial patterns across the cortical surface. However, the principles\ underlying the emergence and utility of functional organization are poorly understood. Here we develop\ the Topographic Deep Artificial Neural Network (TDANN), the first unified model to accurately predict the\ functional organization of multiple cortical areas in the primate visual system. We analyze the key factorsresponsible for the TDANN\’s success and find that it strikes a balance between two specific objectives:achieving a task-general sensory representation that is self-supervised, and maximizing the smoothness of\ responses across the cortical sheet according to a metric that scales relative to cortical surface area. In\ turn, the representations learned by the TDANN are lower dimensional and more brain-like than those in\ models that lack a spatial smoothness constraint. Finally, we provide evidence that the TDANN\’s functionalorganization balances performance with inter-area connection length, and use the resulting models for\ a proof-of-principle optimization of cortical prosthetic design. Our results thus offer a unified principle\ for understanding functional organization and a novel view of the functional role of the visual system in\ particular.

}, doi = { https://doi.org/10.1101/2023.05.18.541361}, url = {https://www.biorxiv.org/content/10.1101/2023.05.18.541361v1.full.pdf}, author = {Margalit, Eshed and Lee, Hyodong and Finzi, Dawn and DiCarlo, James J and Grill-Spector, Kalanit and Yamins, Daniel LK} } @article {170, title = {Adversarially trained neural representations may already be as robust as corresponding biological neural representations}, journal = {arXiv}, year = {2022}, month = {06/19/2022}, type = {preprint}, abstract = {

Visual systems of primates are the gold standard of robust perception. There is thus a general belief that mimicking the neural representations that underlie those systems will yield artificial visual systems that are adversarially robust. In this work, we develop a method for performing adversarial visual attacks directly on primate brain activity. We then leverage this method to demonstrate that the above-mentioned belief might not be well founded. Specifically, we report that the biological neurons that make up visual systems of primates exhibit susceptibility to adversarial perturbations that is comparable in magnitude to existing (robustly trained) artificial neural networks.

}, doi = {https://doi.org/10.48550/arXiv.2206.11228}, url = {https://arxiv.org/abs/2206.11228v1}, author = {Guo, Chong and Lee, Michael J and Leclerc, Guillaume and Dapello, Joel and Rao, Yug and Madry, Aleksander and DiCarlo, James J} } @conference {173, title = {Primate Inferotemporal Cortex Neurons Generalize Better to Novel Image Distributions Than Analogous Deep Neural Networks Units}, booktitle = {SVHRM Workshop at Neural Information Processing Systems (NeurIPS)}, year = {2022}, month = {2022 }, address = {Lisbon, Portugal}, abstract = {

Humans are successfully able to recognize objects in a variety of image distributions. Today\&$\#$39;s artificial neural networks (ANNs), on the other hand, struggle to recognize objects in many image domains, especially those different from the training distribution. It is currently unclear which parts of the ANNs could be improved in order to close this generalization gap. In this work, we used recordings from primate high-level visual cortex (IT) to isolate whether ANNs lag behind primate generalization capabilities because of their encoder (transformations up to the penultimate layer), or their decoder (linear transformation into class labels). Specifically, we fit a linear decoder on images from one domain and evaluate transfer performance on twelve held-out domains, comparing fitting on primate IT representations vs. representations in ANN penultimate layers. To fairly compare, we scale the number of each ANN\&$\#$39;s units so that its in-domain performance matches that of the sampled IT population (i.e. 71 IT neural sites, 73\% binary-choice accuracy). We find that the sampled primate population achieves, on average, 68\% performance on the held-out-domains. Comparably sampled populations from ANN model units generalize less well, maintaining on average 60\%. This is independent of the number of sampled units: models\&$\#$39; out-of-domain accuracies consistently lag behind primate IT. These results suggest that making ANN model units more like primate IT will improve the generalization performance of ANNs.

}, url = {https://openreview.net/pdf?id=iPF7mhoWkOl}, author = {Bagus, Ayu Marliawaty I Gusti and Marques, Tiago and Sanghavi, Sachi and DiCarlo, James J and Schrimpf, Martin} } @conference {146, title = {Wiring Up Vision: Minimizing Supervised Synaptic Updates Needed to Produce a Primate Ventral Stream}, booktitle = {International Conference on Learning Representations 2022 Spotlight}, year = {2022}, month = {April 25, 2022}, type = {preprint}, abstract = {

After training on large datasets, certain deep neural networks are surprisingly good models of the neural mechanisms of adult primate visual object recognition. Nevertheless, these models are considered poor models of the development of the visual system because they posit millions of sequential, precisely coordinated synaptic updates, each based on a labeled image. While ongoing research is pursuing the use of unsupervised proxies for labels, we here explore a complementary strategy of reducing the required number of supervised synaptic updates to produce an adult-like ventral visual stream (as judged by the match to V1, V2, V4, IT, and behavior). Such models might require less precise machinery and energy expenditure to coordinate these updates and would thus move us closer to viable neuroscientific hypotheses about how the visual system wires itself up. Relative to standard model training on labeled images in ImageNet, we here demonstrate that the total number of supervised weight updates can be substantially reduced using three complementary strategies: First, we find that only 2\% of supervised updates (epochs and images) are needed to achieve \∼80\% of a fully trained model\’s match to adult ventral stream. Specifically, training benefits predictions of higher visual cortex the most whereas predictions of earlier areas improve only marginally over the course of training. Second, by improving the random distribution of synaptic connectivity, we find that 54\% of the brain match can already be achieved \“at birth\” (i.e. no training at all). Third, we find that, by training only \∼5\% of model synapses, we can still achieve nearly 80\% of the match to the ventral stream. This approach further improves on ImageNet performance over previous attempts in computer vision of minimizing trained components without substantially increasing the number of trained parameters. These results reflect first steps in modeling not just primate adult visual processing during inference, but also how the ventral visual stream might be \“wired up\” by evolution (a model\’s \“birth\” state) and by developmental learning (a model\’s updates based on visual experience).

}, keywords = {biologically plausible learning, computational neuroscience, convolutional neural networks, primate visual ventral stream}, doi = {10.1101/2020.06.08.140111}, url = {https://openreview.net/pdf?id=g1SzIRLQXMM}, author = {Geiger, Franziska and Schrimpf, Martin and Marques, Tiago and DiCarlo, James J} } @proceedings {163, title = {Combining Different V1 Brain Model Variants to Improve Robustness to Image Corruptions in CNNs}, journal = {Shared Visual Representations in Human \& Machine Intelligence - NeurIPS Workshop}, year = {2021}, month = {October 20, 2021}, publisher = {Neural Information Processing Systems}, abstract = {

While some convolutional neural networks (CNNs) have surpassed human visual abilities in object classification, they often struggle to recognize objects in images corrupted with different types of common noise patterns, highlighting a major limitation of this family of models. Recently, it has been shown that simulating a primary visual cortex (V1) at the front of CNNs leads to small improvements in robustness to these image perturbations. In this study, we start with the observation that different variants of the V1 model show gains for specific corruption types. We then build a new model using an ensembling technique, which combines multiple individual models with different V1 front-end variants. The model ensemble leverages the strengths of each individual model, leading to significant improvements in robustness across all corruption categories and outperforming the base model by 38\% on average. Finally, we show that using distillation, it is possible to partially compress the knowledge in the ensemble model into a single model with a V1 front-end. While the ensembling and distillation techniques used here are hardly biologically-plausible, the results presented here demonstrate that by combining the specific strengths of different neuronal circuits in V1 it is possible to improve the robustness of CNNs for a wide range of perturbations.

}, url = {https://arxiv.org/abs/2110.10645}, author = {Baidya, Avinash and Dapello, Joel and DiCarlo, James J and Marques, Tiago} } @article {161, title = {Computational models of category-selective brain regions enable high-throughput tests of selectivity}, journal = {Nature Communications}, volume = {12}, year = {2021}, month = {Jan-12-2021}, abstract = {

Cortical regions apparently selective to faces, places, and bodies have provided important evidence for domain-specific theories of human cognition, development, and evolution. But claims of category selectivity are not quantitatively precise and remain vulnerable to empirical refutation. Here we develop artificial neural network-based encoding models that accurately predict the response to novel images in the fusiform face area, parahippocampal place area, and extrastriate body area, outperforming descriptive models and experts. We use these models to subject claims of category selectivity to strong tests, by screening for and synthesizing images predicted to produce high responses. We find that these high-responsepredicted images are all unambiguous members of the hypothesized preferred category for each region. These results provide accurate, image-computable encoding models of each category-selective region, strengthen evidence for domain specificity in the brain, and point the way for future research characterizing the functional organization of the brain with unprecedented computational precision.

}, doi = {10.1038/s41467-021-25409-6}, url = {https://www.nature.com/articles/s41467-021-25409-6}, author = {Murty, NAR and Bashivan, Pouya and Abate, Alex and DiCarlo, James J. and Kanwisher, Nancy} } @article {157, title = {Multi-scale hierarchical neural network models that bridge from single neurons in the primate primary visual cortex to object recognition behavior}, journal = {bioRxiv}, year = {2021}, month = {03/01/2021}, type = {preprint}, abstract = {Object recognition relies on inferior temporal (IT) cortical neural population representations that are themselves computed by a hierarchical network of feedforward and recurrently connected neural population called the ventral visual stream (areas V1, V2, V4 and IT). While recent work has created some reasonably accurate image-computable hierarchical neural network models of those neural stages, those models do not yet bridge between the properties of individual neurons and the overall emergent behavior of the ventral stream. For example, current leading ventral stream models do not allow us to ask questions such as: How does the surround suppression behavior of individual V1 neurons ultimately relate to IT neural representation and to behavior?; or How would deactivation of a particular sub-population of V1 neurons specifically alter object recognition behavior? One reason we cannot yet do this is that individual V1 artificial neurons in multi-stage models have not been shown to be functionally similar with individual biological V1 neurons. Here, we took an important first step towards this direction by building and evaluating hundreds of hierarchical neural network models in how well their artificial single neurons approximate macaque primary visual cortical (V1) neurons. We found that single neurons in some models are surprisingly similar to their biological counterparts and that the distributions of single neuron properties, such as those related to orientation and spatial frequency tuning, approximately match those in macaque V1. Crucially, we also observed that hierarchical models with V1-layers that better match macaque V1 at the single neuron level are also more aligned with human object recognition behavior. These results provide the first multi-stage, multi-scale models that allow our field to ask precisely how the specific properties of individual V1 neurons relate to recognition behavior. Finally, we here show that an optimized classical neuroscientific model of V1 is still more functionally similar to primate V1 than all of the tested multi-stage models, suggesting that further model improvements are possible, and that those improvements would likely have tangible payoffs in terms of behavioral prediction accuracy and behavioral robustness.Single neurons in some image-computable hierarchical neural network models are functionally similar to single neurons in macaque primate visual cortex (V1)Some hierarchical neural networks models have V1 layers that better match the biological distributions of macaque V1 single neuron response propertiesMulti-stage hierarchical neural network models with V1 stages that better match macaque V1 are also more aligned with human object recognition behavior at their output stageCompeting Interest StatementThe authors have declared no competing interest.}, doi = {10.1101/2021.03.01.433495}, author = {Tiago Marques and Martin Schrimpf and James J DiCarlo} } @conference {169, title = {Neural Population Geometry Reveals the Role of Stochasticity in Robust Perception}, booktitle = {Neural Information Processing Systems (NeurIPS)}, year = {2021}, address = {Lisbon, Portugal}, abstract = {

Adversarial examples are often cited by neuroscientists and machine learning researchers as an example of how computational models diverge from biological sensory systems. Recent work has proposed adding biologically-inspired components to visual neural networks as a way to improve their adversarial robustness. One surprisingly effective component for reducing adversarial vulnerability is response stochasticity, like that exhibited by biological neurons. Here, using recently developed geometrical techniques from computational neuroscience, we investigate how adversarial perturbations influence the internal representations of standard, adversarially trained, and biologically-inspired stochastic networks. We find distinct geometric signatures for each type of network, revealing different mechanisms for achieving robust representations. Next, we generalize these results to the auditory domain, showing that neural stochasticity also makes auditory models more robust to adversarial perturbations. Geometric analysis of the stochastic networks reveals overlap between representations of clean and adversarially perturbed stimuli, and quantitatively demonstrate that competing geometric effects of stochasticity mediate a tradeoff between adversarial and clean performance. Our results shed light on the strategies of robust perception utilized by adversarially trained and stochastic networks, and help explain how stochasticity may be beneficial to machine and biological computation.

}, url = {https://proceedings.neurips.cc/paper/2021/file/8383f931b0cefcc631f070480ef340e1-Paper.pdf}, author = {Joel Dapello and Jenelle Feather and Tiago Marques and David Cox and Josh McDermott and James J DiCarlo and Sueyeon Chung} } @article {158, title = {The ThreeDWorld Transport Challenge: A Visually Guided Task-and-Motion Planning Benchmark for Physically Realistic Embodied AI}, journal = {arXiv}, year = {2021}, month = {03/25/2021}, type = {preprint}, abstract = {

We introduce a visually-guided and physics-driven taskand-motion planning benchmark, which we call the ThreeDWorld Transport Challenge. In this challenge, an embodied agent equipped with two 9-DOF articulated arms is spawned randomly in a simulated physical home environment. The agent is required to find a small set of objects scattered around the house, pick them up, and transport them to a desired final location. We also position containers around the house that can be used as tools to assist with transporting objects efficiently. To complete the task, an embodied agent must plan a sequence of actions to change the state of a large number of objects in the face of realistic physical constraints. We build this benchmark challenge using the ThreeDWorld simulation: a virtual 3D environment where all objects respond to physics, and where can be controlled using a fully physics-driven navigation and interaction API. We evaluate several existing agents on this benchmark. Experimental results suggest that: 1) a pure RL model struggles on this challenge; 2) hierarchical planningbased agents can transport some objects but still far from solving this task. We anticipate that this benchmark will empower researchers to develop more intelligent physicsdriven robots for the physical world.\ 

}, doi = {arXiv:2103.14025}, url = {https://arxiv.org/abs/2103.14025}, author = {Gan, Chuang and Zhou, Siyuan and Schwartz, Jeremy and Alter, Seth and Bhandwaldar, Abhishek and Gutfreund, Dan and Yamins, Daniel LK and DiCarlo, James J and McDermott, Josh and Torralba, Antonio} } @proceedings {156, title = {Correlation-based spatial layout of deep neural network features generates ventral stream topography}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2020}, month = {2/28/2020}, publisher = {COSYNE}, address = {Denver, CO}, abstract = {

The primate visual system is organized into functional maps, including pinwheel-like arrangements of orientationtuned neurons in primary visual cortex (V1) and patches of category-selective neurons in higher visual cortex. Recent work has demonstrated that deep convolutional neural networks (DCNNs) trained for object recognition are good descriptors of neural representations throughout the ventral pathway, with early, intermediate, and late cortical brain areas best predicted by corresponding layers of the DCNN. Despite this success, DCNNs have no inherent spatial layout for features at a given retinotopic location, and thus, make no predictions regarding many of the characteristic topographic phenomena observed in the brain beyond retinotopy itself, e.g., pinwheels and patches. Cortical map formation has been modeled using self-organizing maps that leverage principles of wiring-length minimization and local correlations of unit responses to produce topographic structure. However, these methods rely on simplified feature parameterizations that limit their ability to accommodate more realistic descriptions of neuron response properties, especially in higher visual areas. Here, we augment DCNNs by assigning model units spatial positions in a 2D \“cortical sheet\” and introduce a novel algorithm to arrange units so that local response correlations are maximized. Applying this algorithm to a categorization-optimized DCNN, we find that layouts generated from earlier layers recapitulate core features of V1 orientation, spatial frequency, and color preference maps, while those generated from later layers naturally exhibit category-selective clusters. Because this wide range of apparently disparate phenomenology is produced by the same underlying principle, our results suggest that the functional architecture of the visual system can be explained by two fundamental constraints: the need to perform visual tasks and the pressure to minimize biophysical costs such as wiring length. Our framework for spatially mapping DCNNs integrates biophysical and representational phenomenology, allowing a more unified understanding of the visual system\’s functional architecture.

}, url = {http://cosyne.org/cosyne20/Cosyne2020_program_book.pdf}, author = {Eshed Margalit and Hyodong Lee and Tiago Marques and James J. DiCarlo and Daniel L.K. Yamins} } @article {147, title = {Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence}, journal = {Neuron}, year = {2020}, month = {Jan-09-2020}, abstract = {

A potentially organizing goal of the brain and cognitive sciences is to accurately explain domains of human intelligence as executable, neurally mechanistic models. Years of research have led to models that capture experimental results in individual behavioral tasks and individual brain regions. We here advocate for taking the next step: integrating experimental results from many laboratories into suites of benchmarks that, when considered together, push mechanistic models toward explaining entire domains of intelligence, such as vision, language, and motor control. Given recent successes of neurally mechanistic models and the surging availability of neural, anatomical, and behavioral data, we believe that now is the time to create integrative benchmarking platforms that incentivize ambitious, unified models. This perspective discusses the advantages and the challenges of this approach and proposes specific steps to achieve this goal in the domain of visual intelligence with the case study of an integrative benchmarking platform called Brain-Score.

}, issn = {08966273}, doi = {10.1016/j.neuron.2020.07.040}, url = {https://linkinghub.elsevier.com/retrieve/pii/S089662732030605X}, author = {Schrimpf, Martin and Kubilius, Jonas and Lee, Michael J. and Murty, NAR and Ajemian, Robert and DiCarlo, James J.} } @article {152, title = {An Open Resource for Non-human Primate Optogenetics}, journal = {Neuron}, year = {2020}, month = {January 10, 2020}, type = {NeuroResource}, abstract = {

Optogenetics has revolutionized neuroscience in small laboratory animals, but its effect on animal models more closely related to humans, such as non-human primates (NHPs), has been mixed. To make evidence-based decisions in primate optogenetics, the scientific community would benefit from a centralized database listing all attempts, successful and unsuccessful, of using optogenetics in the primate brain. We contacted members of the community to ask for their contributions to an open science initiative. As of this writing, 45 laboratories around the world contributed more than 1,000 injection experiments, including precise details regarding their methods and outcomes. Of those entries, more than half had not been published. The resource is free for everyone to consult and contribute to on the Open Science Framework website. Here we review some of the insights from this initial release of the database and discuss methodological considerations to improve the success of optogenetic experiments in NHPs.

}, issn = {08966273}, doi = {10.1016/j.neuron.2020.09.027}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0896627320307510}, author = {Tremblay, Sebastien and Acker, Leah and Afraz, Arash and Albaugh, Daniel L. and Amita, Hidetoshi and Andrei, Ariana R. and Angelucci, Alessandra and Aschner, Amir and Balan, Puiu F. and Basso, Michele A. and Benvenuti, Giacomo and Bohlen, Martin O. and Caiola, Michael J. and Calcedo, Roberto and Cavanaugh, James and Chen, Yuzhi and Chen, Spencer and Chernov, Mykyta M. and Clark, Andrew M. and Dai, Ji and Debes, Samantha R. and Deisseroth, Karl and Desimone, Robert and Dragoi, Valentin and Egger, Seth W. and Eldridge, Mark A.G. and El-Nahal, Hala G. and Fabbrini, Francesco and Federer, Frederick and Fetsch, Christopher R. and Fortuna, Michal G. and Friedman, Robert M. and Fujii, Naotaka and Gail, Alexander and Galvan, Adriana and Ghosh, Supriya and Gieselmann, Marc Alwin and Gulli, Roberto A. and Hikosaka, Okihide and Hosseini, Eghbal A. and Hu, Xing and {\"u}er, Janina and Inoue, Ken-ichi and Janz, Roger and Jazayeri, Mehrdad and Jiang, Rundong and Ju, Niansheng and Kar, Kohitij and Klein, Carsten and Kohn, Adam and Komatsu, Misako and Maeda, Kazutaka and Martinez-Trujillo, Julio C. and Matsumoto, Masayuki and Maunsell, John H.R. and Mendoza-Halliday, Diego and Monosov, Ilya E. and Muers, Ross S. and Nurminen, Lauri and Ortiz-Rios, Michael and {\textquoteright}Shea, Daniel J. and Palfi, {\'e}phane and Petkov, Christopher I. and Pojoga, Sorin and Rajalingham, Rishi and Ramakrishnan, Charu and Remington, Evan D. and Revsine, Cambria and Roe, Anna W. and Sabes, Philip N. and Saunders, Richard C. and Scherberger, {\"o}rg and Schmid, Michael C. and Schultz, Wolfram and Seidemann, Eyal and Senova, Yann-Suhan and Shadlen, Michael N. and Sheinberg, David L. and Siu, Caitlin and Smith, Yoland and Solomon, Selina S. and Sommer, Marc A. and Spudich, John L. and Stauffer, William R. and Takada, Masahiko and Tang, Shiming and Thiele, Alexander and Treue, Stefan and Vanduffel, Wim and Vogels, Rufin and Whitmire, Matthew P. and Wichmann, Thomas and Wurtz, Robert H. and Xu, Haoran and Yazdan-Shahmorad, Azadeh and Shenoy, Krishna V. and DiCarlo, James J. and Platt, Michael L.} } @article {151, title = {Simulating a Primary Visual Cortex at the Front of CNNs Improves Robustness to Image Perturbations}, journal = {Neural Information Processing Systems (NeurIPS; spotlight)}, year = {2020}, month = {June 17, 2020}, type = {preprint}, abstract = {

Current state-of-the-art object recognition models are largely based on convolutional neural network (CNN) architectures, which are loosely inspired by the primate visual system. However, these CNNs can be fooled by imperceptibly small, explicitly crafted perturbations, and struggle to recognize objects in corrupted images that are easily recognized by humans. Here, by making comparisons with primate neural data, we first observed that CNN models with a neural hidden layer that better matches primate primary visual cortex (V1) are also more robust to adversarial attacks. Inspired by this observation, we developed VOneNets, a new class of hybrid CNN vision models. Each VOneNet contains a fixed weight neural network front-end that simulates primate V1, called the VOneBlock, followed by a neural network back-end adapted from current CNN vision models. The VOneBlock is based on a classical neuroscientific model of V1: the linear-nonlinear-Poisson model, consisting of a biologically-constrained Gabor filter bank, simple and complex cell nonlinearities, and a V1 neuronal stochasticity generator. After training, VOneNets retain high ImageNet performance, but each is substantially more robust, outperforming the base CNNs and state-of-the-art methods by 18\% and 3\%, respectively, on a conglomerate benchmark of perturbations comprised of white box adversarial attacks and common image corruptions. Finally, we show that all components of the VOneBlock work in synergy to improve robustness. While current CNN architectures are arguably brain-inspired, the results presented here demonstrate that more precisely mimicking just one stage of the primate visual system leads to new gains in ImageNet-level computer vision applications.

}, doi = {10.1101/2020.06.16.154542}, url = {https://www.biorxiv.org/content/10.1101/2020.06.16.154542v27}, author = {Dapello, Joel and Marques, Tiago and Schrimpf, Martin and Geiger, Franziska and Cox, David D and DiCarlo, James J} } @article {148, title = {ThreeDWorld: A Platform for Interactive Multi-Modal Physical Simulation}, journal = {arXiv}, year = {2020}, month = {July 9, 2020}, type = {preprint}, abstract = {

We introduce ThreeDWorld (TDW), a platform for interactive multi-modal physical simulation. With TDW, users can simulate high-fidelity sensory data and physical interactions between mobile agents and objects in a wide variety of rich 3D environments. TDW has several unique properties: 1) realtime near photo-realistic image rendering quality; 2) a library of objects and environments with materials for high-quality rendering, and routines enabling user customization of the asset library; 3) generative procedures for efficiently building classes of new environments 4) high-fidelity audio rendering; 5) believable and realistic physical interactions for a wide variety of material types, including cloths, liquid, and deformable objects; 6) a range of \"avatar\" types that serve as embodiments of AI agents, with the option for user avatar customization; and 7) support for human interactions with VR devices. TDW also provides a rich API enabling multiple agents to interact within a simulation and return a range of sensor and physics data representing the state of the world. We present initial experiments enabled by the platform around emerging research directions in computer vision, machine learning, and cognitive science, including multi-modal physical scene understanding, multi-agent interactions, models that \"learn like a child\", and attention studies in humans and neural networks. The simulation platform will be made publicly available.

}, url = {https://arxiv.org/abs/2007.04954}, author = {Gan, Chuang and Schwartz, Jeremy and Alter, Seth and Schrimpf, Martin and Traer, James and De Freitas, Julian and Kubilius, Jonas and Bhandwaldar, Abhishek and Haber, Nick and Sano, Megumi and Wang, Elias and Mrowca, Damian and Lingelbach, Michael and Curtis, Aidan and Figelis, Kevin and Bear, Daniel M. and Gutfreund, Dan and Cox, David and DiCarlo, James J. and McDermott, Josh and Tenenbaum, Joshua B. and Yamins, Daniel L.K.} } @article {5, title = {Topographic deep artificial neural networks reproduce the hallmarks of the primate inferior temporal cortex face processing network}, journal = {bioRxiv}, year = {2020}, month = {07/2020}, type = {preprint}, abstract = {

A salient characteristic of monkey inferior temporal (IT) cortex is the IT face processing network. Its hallmarks include: \“face neurons\” that respond more to faces than non-face objects, strong spatial clustering of those neurons in foci at each IT anatomical level (\“face patches\”), and the preferential interconnection of those foci. While some deep artificial neural networks (ANNs) are good predictors of IT neuronal responses, including face neurons, they do not explain those face network hallmarks. Here we ask if they might be explained with a simple, metabolically motivated addition to current ANN ventral stream models. Specifically, we designed and successfully trained topographic deep ANNs (TDANNs) to solve real-world visual recognition tasks (as in prior work), but, in addition, we also optimized each network to minimize a proxy for neuronal wiring length within its IT layers. We report that after this dual optimization, the model IT layers of TDANNs reproduce the hallmarks of the IT face network: the presence of face neurons, clusters of face neurons that quantitatively match those found in IT face patches, connectivity between those patches, and the emergence of face viewpoint invariance along the network hierarchy. We find that these phenomena emerge for a range of naturalistic experience, but not for highly unnatural training. Taken together, these results show that the IT face processing network could be a consequence of a basic hierarchical anatomy along the ventral stream, selection pressure on the visual system to accomplish general object categorization, and selection pressure to minimize axonal wiring length.

}, doi = {https://doi.org/10.1101/2020.07.09.185116}, url = {https://www.biorxiv.org/content/10.1101/2020.07.09.185116v1.full.pdf}, author = {Hyodong Lee and Eshed Margalit and Kamila M. Jozwik and Michael A. Cohen and Nancy Kanwisher and Daniel L. K. Yamins and James J. DiCarlo} } @conference {11, title = {Brain-Like Object Recognition with High-Performing Shallow Recurrent ANNs}, booktitle = {Neural Information Processing Systems}, year = {2019}, abstract = {

Deep convolutional artificial neural networks (ANNs) are the leading class of candidate models of the mechanisms of visual processing in the primate ventral stream. While initially inspired by brain anatomy, over the past years, these ANNs have evolved from a simple eight-layer architecture in AlexNet to extremely deep and branching architectures, demonstrating increasingly better object categorization performance, yet bringing into question how brain-like they still are. In particular, typical deep models from the machine learning community are often hard to map onto the brain\&$\#$39;s anatomy due to their vast number of layers and missing biologically-important connections, such as recurrence. Here we demonstrate that better anatomical alignment to the brain and high performance on machine learning as well as neuroscience measures do not have to be in contradiction. We developed CORnet-S, a shallow ANN with four anatomically mapped areas and recurrent connectivity, guided by Brain-Score, a new large-scale composite of neural and behavioral benchmarks for quantifying the functional fidelity of models of the primate ventral visual stream. Despite being significantly shallower than most models, CORnet-S is the top model on Brain-Score and outperforms similarly compact models on ImageNet. Moreover, our extensive analyses of CORnet-S circuitry variants reveal that recurrence is the main predictive factor of both Brain-Score and ImageNet top-1 performance. Finally, we report that the temporal evolution of the CORnet-S \"IT\" neural population resembles the actual monkey IT population dynamics. Taken together, these results establish CORnet-S, a compact, recurrent ANN, as the current best model of the primate ventral visual stream.

}, doi = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns}, url = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns.pdf}, author = {Jonas Kubilius and Martin Schrimpf and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kohitij Kar and Bashivan, Pouya and Jonathan Prescott-Roy and Kailyn Schmidt and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and James J. DiCarlo} } @inbook {12, title = {Funcitional Properties of Circuits, Cellular Populations, and Areas}, booktitle = {The Neocortex}, volume = {27}, year = {2019}, pages = {223-265}, publisher = {The MIT Press}, organization = {The MIT Press}, chapter = {13}, address = {Cambridge, MA}, issn = {978-0-262-04324-3}, doi = {10.7551/mitpress/12593.001.0001}, url = {http://www.esforum.de}, author = {Harris, KD and Groh, JM and DiCarlo, JJ and Fries, P and Kaschube, M and Laurent, G and MacLean, JN and McCormick, DA and Pipa, G and Reynolds, JN and Schwartz, AB and Sejnowksi, TJ and Singer, W and Vinck, M}, editor = {Singer, Wolf and Sejnowski, Terrence J. and Rakic, Pasko} } @conference {166, title = {Using Brain-Score to Evaluate and Build Neural Networks for Brain-Like Object Recognition}, booktitle = {Computational and Systems Neuroscience (COSYNE)}, year = {2019}, address = {Denver, CO}, author = {Schrimpf, Martin and Kubilius, Jonas and Hong, Ha and Majaj, Najib and Rajalingham, Rishi and Issa, Elias B and Kar, Kohitij and Ziemba, Corey M and Bashivan, Pouya and Prescott-Roy, Jonathan and Schmidt, Kailyn and Yamins, Daniel LK and DiCarlo, James J} } @article {22, title = {Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?}, journal = {bioRxiv}, year = {2018}, month = {09/2018}, type = {preprint}, abstract = {

The internal representations of early deep artificial neural networks (ANNs) were found to be remarkably similar to the internal neural representations measured experimentally in the primate brain. Here we ask, as deep ANNs have continued to evolve, are they becoming more or less brain-like? ANNs that are most functionally similar to the brain will contain mechanisms that are most like those used by the brain. We therefore developed Brain-Score - a composite of multiple neural and behavioral benchmarks that score any ANN on how similar it is to the brain\&$\#$39;s mechanisms for core object recognition - and we deployed it to evaluate a wide range of state-of-the-art deep ANNs. Using this scoring system, we here report that: (1) DenseNet-169, CORnet-S and ResNet-101 are the most brain-like ANNs. (2) There remains considerable variability in neural and behavioral responses that is not predicted by any ANN, suggesting that no ANN model has yet captured all the relevant mechanisms. (3) Extending prior work, we found that gains in ANN ImageNet performance led to gains on Brain-Score. However, correlation weakened at \>= 70\% top-1 ImageNet performance, suggesting that additional guidance from neuroscience is needed to make further advances in capturing brain mechanisms. (4) We uncovered smaller (i.e. less complex) ANNs that are more brain-like than many of the best-performing ImageNet models, which suggests the opportunity to simplify ANNs to better understand the ventral stream. The scoring system used here is far from complete. However, we propose that evaluating and tracking model-benchmark correspondences through a Brain-Score that is regularly updated with new brain data is an exciting opportunity: experimental benchmarks can be used to guide machine network evolution, and machine networks are mechanistic hypotheses of the brain\&$\#$39;s network and thus drive next experiments. To facilitate both of these, we release Brain-Score.org: a platform that hosts the neural and behavioral benchmarks, where ANNs for visual processing can be submitted to receive a Brain-Score and their rank relative to other models, and where new experimental data can be naturally incorporated.

}, doi = {https://doi.org/10.1101/407007}, url = {https://www.biorxiv.org/content/10.1101/407007v2.full.pdf}, author = {Martin Schrimpf and Kubilius, Jonas and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kar, Kohitij and Bashivan, Pouya and Jonathan Prescott-Roy and Schmidt, Kailyn and Daniel L. K. Yamins and DiCarlo, James J.} } @article {27, title = {Explicit information for category-orthogonal object properties increases along the ventral stream}, journal = {Nature Neuroscience}, volume = {19}, year = {2016}, month = {02/2016}, pages = {613 - 622}, abstract = {

Extensive research has revealed that the ventral visual stream hierarchically builds a robust representation for supporting visual object categorization tasks. We systematically explored the ability of multiple ventral visual areas to support a variety of \&$\#$39;category-orthogonal\&$\#$39; object properties such as position, size and pose. For complex naturalistic stimuli, we found that the inferior temporal (IT) population encodes all measured category-orthogonal object properties, including those properties often considered to be low-level features (for example, position), more explicitly than earlier ventral stream areas. We also found that the IT population better predicts human performance patterns across properties. A hierarchical neural network model based on simple computational principles generates these same cross-area patterns of information. Taken together, our empirical results support the hypothesis that all behaviorally relevant object properties are extracted in concert up the ventral visual hierarchy, and our computational model explains how that hierarchy might be built.

}, issn = {1097-6256}, doi = {10.1038/nn.4247}, url = {http://www.nature.com/articles/nn.4247}, author = {Hong, Ha and Yamins, Daniel L K and Majaj, Najib J and DiCarlo, James J} } @article {28, title = {Simple Learned Weighted Sums of Inferior Temporal Neuronal Firing Rates Accurately Predict Human Core Object Recognition Performance}, journal = {Journal of Neuroscience}, volume = {35}, year = {2015}, month = {06/2017}, pages = {13402 - 13418}, abstract = {

To go beyond qualitative models of the biological substrate of object recognition, we ask: can a single ventral stream neuronal linking hypothesis quantitatively account for core object recognition performance over a broad range of tasks? We measured human performance in 64 object recognition tests using thousands of challenging images that explore shape similarity and identity preserving object variation. We then used multielectrode arrays to measure neuronal population responses to those same images in visual areas V4 and inferior temporal (IT) cortex of monkeys and simulated V1 population responses. We tested leading candidate linking hypotheses and control hypotheses, each postulating how ventral stream neuronal responses underlie object recognition behavior. Specifically, for each hypothesis, we computed the predicted performance on the 64 tests and compared it with the measured pattern of human performance. All tested hypotheses based on low- and mid-level visually evoked activity (pixels, V1, and V4) were very poor predictors of the human behavioral pattern. However, simple learned weighted sums of distributed average IT firing rates exactly predicted the behavioral pattern. More elaborate linking hypotheses relying on IT trial-by-trial correlational structure, finer IT temporal codes, or ones that strictly respect the known spatial substructures of IT (\"face patches\") did not improve predictive power. Although these results do not reject those more elaborate hypotheses, they suggest a simple, sufficient quantitative model: each object recognition task is learned from the spatially distributed mean firing rates (100 ms) of \∼60,000 IT neurons and is executed as a simple weighted sum of those firing rates.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.5181-14.2015}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.5181-14.2015}, author = {Majaj, N. J. and Hong, H. and Solomon, E. A. and DiCarlo, J. J.} } @article {32, title = {Deep Neural Networks Rival the Representation of Primate IT Cortex for Core Visual Object Recognition}, journal = {PLoS Computational Biology}, volume = {10}, year = {2014}, month = {12/2014}, pages = {e1003963}, abstract = {

The primate visual system achieves remarkable visual object recognition performance even in brief presentations, and under changes to object exemplar, geometric transformations, and background variation (a.k.a. core visual object recognition). This remarkable performance is mediated by the representation formed in inferior temporal (IT) cortex. In parallel, recent advances in machine learning have led to ever higher performing models of object recognition using artificial deep neural networks (DNNs). It remains unclear, however, whether the representational performance of DNNs rivals that of the brain. To accurately produce such a comparison, a major difficulty has been a unifying metric that accounts for experimental limitations, such as the amount of noise, the number of neural recording sites, and the number of trials, and computational limitations, such as the complexity of the decoding classifier and the number of classifier training examples. In this work, we perform a direct comparison that corrects for these experimental limitations and computational considerations. As part of our methodology, we propose an extension of \"kernel analysis\" that measures the generalization accuracy as a function of representational complexity. Our evaluations show that, unlike previous bio-inspired models, the latest DNNs rival the representational performance of IT cortex on this visual object recognition task. Furthermore, we show that models that perform well on measures of representational performance also perform well on measures of representational similarity to IT, and on measures of predicting individual IT multi-unit responses. Whether these DNNs rely on computational mechanisms similar to the primate visual system is yet to be determined, but, unlike all previous bio-inspired models, that possibility cannot be ruled out merely on representational performance grounds.

}, doi = {10.1371/journal.pcbi.1003963}, url = {https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003963\&type=printable}, author = {Cadieu, Charles F. and Hong, Ha and Yamins, Daniel L. K. and Pinto, Nicolas and Ardila, Diego and Solomon, Ethan A. and Majaj, Najib J. and DiCarlo, James J.}, editor = {Bethge, Matthias} } @article {36, title = {The Neural Representation Benchmark and its Evaluation on Brain and Machine}, journal = {arXiv}, year = {2013}, month = {01/2013}, type = {preprint}, abstract = {

A key requirement for the development of effective learning representations is their evaluation and comparison to representations we know to be effective. In natural sensory domains, the community has viewed the brain as a source of inspiration and as an implicit benchmark for success. However, it has not been possible to directly test representational learning algorithms directly against the representations contained in neural systems. Here, we propose a new benchmark for visual representations on which we have directly tested the neural representation in multiple visual cortical areas in macaque (utilizing data from [Majaj et al., 2012]), and on which any computer vision algorithm that produces a feature space can be tested. The benchmark measures the effectiveness of the neural or machine representation by computing the classification loss on the ordered eigendecomposition of a kernel matrix [Montavon et al., 2011]. In our analysis we find that the neural representation in visual area IT is superior to visual area V4. In our analysis of representational learning algorithms, we find that three-layer models approach the representational performance of V4 and the algorithm in [Le et al., 2012] surpasses the performance of V4. Impressively, we find that a recent supervised algorithm [Krizhevsky et al., 2012] achieves performance comparable to that of IT for an intermediate level of image variation difficulty, and surpasses IT at a higher difficulty level. We believe this result represents a major milestone: it is the first learning algorithm we have found that exceeds our current estimate of IT representation performance. We hope that this benchmark will assist the community in matching the representational performance of visual cortex and will serve as an initial rallying point for further correspondence between representations derived in brains and machines.

}, doi = {https://arxiv.org/abs/1301.3530}, url = {https://arxiv.org/pdf/1301.3530.pdf}, author = {Cadieu, Charles F. and Ha Hong and Daniel L. K. Yamins and Pinto, Nicolas and Majaj, Najib J and James J. DiCarlo} } @conference {42, title = {A unified neuronal population code fully explains human object recognition.}, booktitle = {Computation and Systems Neuroscience (COSYNE)}, year = {2012}, month = {02/2012}, address = {Salt Lake City, Utah, USA}, abstract = {

Our goal is to understand the neuronal mechanisms that underlie human visual object recognition (OR). While previous work has argued for qualitative links between neuronal responses in the ventral visual stream and human shape judgements, no study has asked which, if any, neuronal responses are quantitatively sufficient to explain broad domain human OR performance. The shift from qualitative to quantitative hypotheses requires a framework to link neuronal responses to behavior (\“unified code\”). Here we ask: is there a common neuronal basis (e.g., in IT cortex) and a simple (e.g., linear) transformation that will predict all of human OR performance? We first defined OR operationally by obtaining human psychophysical measurements using images that explore shape similarity and identity preserving image variation, resulting in OR benchmarks that span a range of difficulty. Using the same visual images, we measured neuronal responses in V4 and IT in two monkeys. We implemented 14 unified codes based on those neuronal data and computed cross-validated neuronal discriminability indices (d\’s) to compare to the human d\’s. The dynamic range across those d\’s sets a high bar for when a putative code is sufficient to explain behavior: it is not sufficient for a code to perform well (high d\’) or to match one d\’. Instead, a sufficient unified code must also emergently predict the entire pattern of behavior over all tasks. Remarkably, we found a few unified IT-based codes that meet this high bar. Interestingly, many other IT codes and all V4 codes are insufficient. While humans outperform computer vision systems on many of our OR tasks, their abilities reliably depend on the images tested. These dependencies in human performance are fully explained by a simple, unified reading of monkey ventral stream neurons, a feat unmatched by any computer vision system we tested

}, doi = {http://www.cosyne.org/c/index.php?title=Cosyne_12}, url = {http://cosyne.org/cosyne12/Cosyne2012_program_book.pdf}, author = {Majaj, Najib J and Hong, Ha and Solomon, EA and DiCarlo, James J.} } @proceedings {51, title = {Human versus machine: comparing visual object recognition systems on a level playing field}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2010}, month = {02/2010}, address = {Salt Lake City, Utah, USA}, abstract = {

It is received wisdom that biological visual systems easily outmatch current artificial systems at complex visual tasks like object recognition. But have the appropriate comparisons been made? Because artificial systems are improving every day, they may surpass human performance some day. We must understand our progress toward reaching that day, because that success is one of several necessary requirements for \"understanding\" visual object recognition. How large (or small) is the difference in performance between current state-of-the-art object recognition systems and the primate visual system? In practice, the performance comparison of any two object recognition systems requires a focus on the computational crux of the problem and sets of images that engage it. Although it is widely believed that tolerance (\"invariance\") to identity-preserving image variation (e.g. variation in object position, scale, pose, illumination) is critical, systematic comparisons of state-of-the-art artificial visual representations almost always rely on \"natural\" image databases that can fail to probe the ability of a recognition system to solve the invariance problem [Pinto et al PLoS08, COSYNE08, ECCV08, CVPR09]. Thus, to understand how well current state-of-the-art visual representations perform relative to each other, relative to low-level neuronal representations (e.g. retinal-like and V1-like), and relative to high-level representations (e.g. human performance), we tested all of these representations on a common set of visual object recognition tasks that directly engage the invariance problem. Specifically, we used a synthetic testing approach that allows direct engagement of the invariance problem, as well as knowledge and control of all the key parameters that make object recognition challenging. We successfully re-implemented a variety of state-of-the-art visual representations, and we confirmed the high published performance of all of these state-of-the-art representations on large, complex \"natural\â\€ image benchmarks. Surprisingly, we found that most of these representations were weak on our simple synthetic tests of invariant recognition, and only high-level biologically-inspired representations showed performance gains above the neuroscience \"null\" representation (V1-like). While in aggregate, we found that the performance of these state-of-the-art representations pales in comparison to human performance, humans and computers seem to fail in different and potentially enlightening ways when faced with the problem of invariance. We also show how our synthetic testing approach can more deeply illuminate the strengths and weaknesses of different visual representations and thus guide progress on invariant object recognition.

}, doi = {10.3389/conf.fnins.2010.03.00283}, url = {https://www.frontiersin.org/Community/AbstractDetails.aspx?ABS_DOI=10.3389/conf.fnins.2010.03.00283\&eid=770\&sname=Computational_and_Systems_Neuroscience_2010}, author = {Pinto, Nicolas and Majaj, Najib J and Barhomi, Y and Solomon, E. A. and Cox, David D. and DiCarlo, James J.} } @article {90, title = {{\textquoteright}Breaking{\textquoteright} position-invariant object recognition}, journal = {Nature Neuroscience}, volume = {8}, year = {2005}, month = {08/2005}, pages = {1145 - 1147}, abstract = {

While it is often assumed that objects can be recognized irrespective of where they fall on the retina, little is known about the mechanisms underlying this ability. By exposing human subjects to an altered world where some objects systematically changed identity during the transient blindness that accompanies eye movements, we induced predictable object confusions across retinal positions, effectively \&$\#$39;breaking\&$\#$39; position invariance. Thus, position invariance is not a rigid property of vision but is constantly adapting to the statistics of the environment.

}, issn = {1097-6256}, doi = {10.1038/nn1519}, url = {http://www.nature.com/articles/nn1519.pdf}, author = {Cox, David D and Meier, Philip and Oertelt, Nadja and DiCarlo, James J} } @proceedings {97, title = {Mapping functional neuronal processing chains underlying sensory-motor tasks in the primate}, journal = {Gordon Conference: Sensory coding and the natural environment}, year = {2004}, month = {2004}, address = {Oxford, UK}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @article {89, title = {Using Neuronal Latency to Determine Sensory{\textendash}Motor Processing Pathways in Reaction Time Tasks}, journal = {Journal of Neurophysiology}, volume = {93}, year = {2004}, month = {11/2004}, pages = {2974 - 2986}, abstract = {

We describe a new technique that uses the timing of neuronal and behavioral responses to explore the contributions of individual neurons to specific behaviors. The approach uses both the mean neuronal latency and the trial-by-trial covariance between neuronal latency and behavioral response. Reliable measurements of these values were obtained from single-unit recordings made from anterior inferotemporal (AIT) cortex and the frontal eye fields (FEF) in monkeys while they performed a choice reaction time task. These neurophysiological data show that the responses of AIT neurons and some FEF neurons have little covariance with behavioral response, consistent with a largely \"sensory\" response. The responses of another group of FEF neurons with longer mean latency covary tightly with behavioral response, consistent with a largely \"motor\" response. A very small fraction of FEF neurons had responses consistent with an intermediate position in the sensory-motor pathway. These results suggest that this technique is a valuable tool for exploring the functional organization of neuronal circuits that underlie specific behaviors.

}, keywords = {Action Potentials, Afferent, Animal, Animals, Behavior, Macaca mulatta, Male, Models, Motor Neurons, Neural Pathways, Neurological, Neurons, Photic Stimulation, Psychomotor Performance, Reaction Time, Task Performance and Analysis, Temporal Lobe, Time Factors, Visual Fields}, issn = {0022-3077}, doi = {10.1152/jn.00508.2004}, url = {https://www.physiology.org/doi/10.1152/jn.00508.2004}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @article {99, title = {Anterior Inferotemporal Neurons of Monkeys Engaged in Object Recognition Can be Highly Sensitive to Object Retinal Position}, journal = {Journal of Neurophysiology}, volume = {89}, year = {2003}, month = {01/2003}, pages = {3264 - 3278}, abstract = {

Visual object recognition is computationally difficult because changes in an object\&$\#$39;s position, distance, pose, or setting may cause it to produce a different retinal image on each encounter. To robustly recognize objects, the primate brain must have mechanisms to compensate for these variations. Although these mechanisms are poorly understood, it is thought that they elaborate neuronal representations in the inferotemporal cortex that are sensitive to object form but substantially invariant to other image variations. This study examines this hypothesis for image variation resulting from changes in object position. We studied the effect of small differences (+/-1.5 degrees ) in the retinal position of small (0.6 degrees wide) visual forms on both the behavior of monkeys trained to identify those forms and the responses of 146 anterior IT (AIT) neurons collected during that behavior. Behavioral accuracy and speed were largely unaffected by these small changes in position. Consistent with previous studies, many AIT responses were highly selective for the forms. However, AIT responses showed far greater sensitivity to retinal position than predicted from their reported receptive field (RF) sizes. The median AIT neuron showed a approximately 60\% response decrease between positions within +/-1.5 degrees of the center of gaze, and 52\% of neurons were unresponsive to one or more of these positions. Consistent with previous studies, each neuron\&$\#$39;s rank order of target preferences was largely unaffected across position changes. Although we have not yet determined the conditions necessary to observe this marked position sensitivity in AIT responses, we rule out effects of spatial-frequency content, eye movements, and failures to include the RF center. To reconcile this observation with previous studies, we hypothesize that either AIT position sensitivity strongly depends on object size or that position sensitivity is sharpened by extensive visual experience at fixed retinal positions or by the presence of flanking distractors.

}, keywords = {Action Potentials, Animals, Depth Perception, Electrophysiology, Eye Movements, Form Perception, Macaca mulatta, Male, Neurons, Pattern Recognition, Photic Stimulation, Psychomotor Performance, Retina, Temporal Lobe, Time Factors, Visual, Visual Fields, Visual Perception}, issn = {0022-3077}, doi = {10.1152/jn.00358.2002}, url = {https://www.physiology.org/doi/10.1152/jn.00358.2002}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @proceedings {101, title = {Using reaction time tasks to map sensory-motor chains in the monkey}, journal = {Society for Neuroscience}, year = {2002}, month = {10/2002}, publisher = {SFN}, address = {Orlando, FL, USA}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @article {105, title = {Form representation in monkey inferotemporal cortex is virtually unaltered by free viewing}, journal = {Nature Neuroscience}, volume = {3}, year = {2000}, month = {01/2000}, pages = {814 - 821}, abstract = {

How are objects represented in the brain during natural behavior? Visual object recognition in primates is thought to depend on the inferotemporal cortex {(IT).} In most neurophysiological studies of {IT,} monkeys hold their direction of gaze fixed while isolated visual stimuli are presented (controlled viewing). However, during natural behavior, primates visually explore cluttered environments by changing gaze direction several times each second (free viewing). We examined the effect of free viewing on {IT} neuronal responses in monkeys engaged in a form-recognition task. By making small, real-time stimulus adjustments, we produced nearly identically retinal stimulation during controlled and free viewing. Nearly 90\% of neuronal responses were unaffected by free viewing, and average stimulus selectivity was unchanged. Thus, neuronal representations that likely underlie form recognition are virtually unaltered by free viewing.

}, keywords = {Animals, Conditioning, Fixation, Form Perception, Macaca mulatta, Male, Neurons, Ocular, Pattern Recognition, Photic Stimulation, Psychology, Saccades, Temporal Lobe, Visual, Visual Cortex}, issn = {1097-6256}, doi = {10.1038/77722}, url = {http://www.nature.com/articles/nn0800_814}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @proceedings {106, title = {Inferotemporal representations underlying object recognition in the free viewing monkey}, journal = {Society for Neuroscience}, year = {2000}, month = {10/2000}, publisher = {SFN}, address = {New Orleans, LA, USA}, author = {DiCarlo, James J. and Maunsell, John H. R.} }