@booklet {185, title = {How does the primate brain combine generative and discriminative computations in vision?}, year = {2024}, abstract = {

Vision is widely understood as an inference problem. However, two contrasting conceptions of the inference process have each been influential in research on biological vision as well as the engineering of machine vision. The first emphasizes bottom-up signal flow, describing vision as a largely feedforward, discriminative inference process that filters and transforms the visual information to remove irrelevant variation and represent behaviorally relevant information in a format suitable for downstream functions of cognition and behavioral control. In this conception, vision is driven by the sensory data, and perception is direct because the processing proceeds from the data to the latent variables of interest. The notion of \"inference\" in this conception is that of the engineering literature on neural networks, where feedforward convolutional neural networks processing images are said to perform inference. The alternative conception is that of vision as an inference process in Helmholtz\&$\#$39;s sense, where the sensory evidence is evaluated in the context of a generative model of the causal processes giving rise to it. In this conception, vision inverts a generative model through an interrogation of the evidence in a process often thought to involve top-down predictions of sensory data to evaluate the likelihood of alternative hypotheses. The authors include scientists rooted in roughly equal numbers in each of the conceptions and motivated to overcome what might be a false dichotomy between them and engage the other perspective in the realm of theory and experiment. The primate brain employs an unknown algorithm that may combine the advantages of both conceptions. We explain and clarify the terminology, review the key empirical evidence, and propose an empirical research program that transcends the dichotomy and sets the stage for revealing the mysterious hybrid algorithm of primate vision.

}, keywords = {discriminative model, generativemodel, Primatevision, visual inference}, doi = { https://doi.org/10.48550/arXiv.2401.06005}, url = {https://arxiv.org/pdf/2401.06005.pdf}, author = {Benjamin Peters and James J. DiCarlo and Todd Gureckis and Ralf Haefner and Leyla Isik and Joshua Tenenbaum and Talia Konkle and Thomas Naselaris and Kimberly Stachenfeld and Zenna Tavares and Doris Tsao and Ilker Yildirim and Nikolaus Kriegeskorte} } @article {188, title = {Software and Methods for Controlling Neural Responses in Deep Brain Regions}, year = {2024}, chapter = {US20240082534A1}, abstract = {

Techniques for non-invasively controlling targeted neural activity of a subject are provided herein. The techniques include applying a stimulus input to the subject, the stimulus input being formed by a deep artificial neural network (ANN) model and being configured to elicit targeted neural activity within a brain of the subject. The stimulus input may be a pattern of luminous power generated by the deep ANN model and applied to retinae of the subject. The stimulus input may be generated by the deep ANN model based on a mapping of the subject\&$\#$39;s neural responses to neurons of the deep ANN model.

}, issn = {US20240082534A1}, url = {https://patents.google.com/patent/US20240082534A1/en}, author = {DiCarlo, James and Bashivan, Pouya and Kar, Kohitij} } @article {174, title = {Catalyzing next-generation Artificial Intelligence through NeuroAI}, journal = {Nature Communications}, volume = {14}, year = {2023}, month = {Jan-12-2023}, pages = {1597}, abstract = {

Neuroscience has long been an essential driver of progress in artificial intelligence (AI). We propose that to accelerate progress in AI, we must invest in fundamental research in NeuroAI. A core component of this is the embodied Turing test, which challenges AI animal models to interact with the sensorimotor world at skill levels akin to their living counterparts. The embodied Turing test shifts the focus from those capabilities like game playing and language that are especially well-developed or uniquely human to those capabilities \– inherited from over 500 million years of evolution \– that are shared with all animals. Building models that can pass the embodied Turing test will provide a roadmap for the next generation of AI.

}, doi = {10.1038/s41467-023-37180-x}, url = {https://www.nature.com/articles/s41467-023-37180-xhttps://www.nature.com/articles/s41467-023-37180-x.pdfhttps://www.nature.com/articles/s41467-023-37180-x.pdfhttps://www.nature.com/articles/s41467-023-37180-x}, author = {Zador, Anthony and Escola, Sean and Richards, Blake and {\"O}lveczky, Bence and Bengio, Yoshua and Boahen, Kwabena and Botvinick, Matthew and Chklovskii, Dmitri and Churchland, Anne and Clopath, Claudia and DiCarlo, James and Ganguli, Surya and Hawkins, Jeff and {\"o}rding, Konrad and Koulakov, Alexei and LeCun, Yann and Lillicrap, Timothy and Marblestone, Adam and Olshausen, Bruno and Pouget, Alexandre and Savin, Cristina and Sejnowski, Terrence and Simoncelli, Eero and Solla, Sara and Sussillo, David and Tolias, Andreas S. and Tsao, Doris} } @article {178, title = {fROI-level computational models enable broad-scale experimental testing and expose key divergences between models and brains}, journal = {Journal of Vision}, volume = {23}, year = {2023}, month = {2023}, pages = {5788 - 5788}, abstract = {

Deep convolutional neural network (DNN)- based models have emerged as our leading hypotheses of human vision. Here we describe, and expand upon, our latest effort to use DNN models of brain regions to explain key results from previous cognitive neuroscience and psychology experiments. Many stimuli in these prior experiments were highly manipulated (e.g. scrambled body parts, face parts, re-arranged spatial positions) often outside the domain of natural stimuli. These results can therefore be considered as tests of model generalization beyond naturalistic stimuli. We first performed these tests on the fusiform face area (FFA), parahippocampal place area (PPA) and the extrastriate body area (EBA). Our previous results (presented in VSS2022) showed that our fROI-level models recapitulate several key results from prior studies. We also observed that models did not perform as well on non-naturalistic stimuli. Here we extend our model evaluation metrics in two ways. First, we replicated findings from the original paper on the EBA (Downing et al 2001) that the EBA responds as highly to line drawings of bodies, and symbolic stick figures as to natural images of bodies (and not to control conditions like faces and objects). Second, we find that none of the computational models explain this pattern of observed responses, though models trained with language-based supervision (like CLIP) do better than other models. Together, our results on symbolic body images expose the bounds of current computational models. This progress was made possible only because of fROI-level modeling procedures, and opens up new ways to understand the power and limitations of current models and test novel hypotheses completely in-silico.

}, isbn = {1534-7362}, url = {https://doi.org/10.1167/jov.23.9.5788}, author = {Mieczkowski, Elizabeth and Abate, Alex and De Faria, Willian and Lydic, Kirsten and DiCarlo, James and Kanwisher, Nancy and Apurva Ratan Murty, N.} } @article {183, title = {How well do rudimentary plasticity rules predict adult visual object learning?}, journal = {PLOS Computational Biology}, volume = {19}, year = {2023}, month = {Nov-12-2023}, pages = {e1011713}, abstract = {

A core problem in visual object learning is using a finite number of images of a new object to accurately identify that object in future, novel images.

One longstanding, conceptual hypothesis asserts that this core problem is solved by adult brains through two connected mechanisms: 1) the re-representation of incoming retinal images as points in a fixed, multidimensional neural space, and 2) the optimization of linear decision boundaries in that space, via simple plasticity rules applied to a single downstream layer.

Though this scheme is biologically plausible, the extent to which it explains learning behavior in humans has been unclear\—in part because of a historical lack of image-computable models of the putative neural space, and in part because of a lack of measurements of human learning behaviors in difficult, naturalistic settings.

Here, we addressed these gaps by 1) drawing from contemporary, image-computable models of the primate ventral visual stream to create a large set of testable learning models (n = 2,408 models), and 2) using online psychophysics to measure human learning trajectories over a varied set of tasks involving novel 3D objects (n = 371,000 trials), which we then used to develop (and\ publicly release) empirical benchmarks for comparing learning models to humans.

We evaluated each learning model on these benchmarks, and found those based on deep, high-level representations from neural networks were surprisingly aligned with human behavior. While no tested model explained the entirety of replicable human behavior, these results establish that rudimentary plasticity rules, when combined with appropriate visual representations, have high explanatory power in predicting human behavior with respect to this core object learning problem.

}, issn = {1553-734X}, doi = {10.1371/journal.pcbi.1011713}, url = {https://dx.plos.org/10.1371/journal.pcbi.1011713}, author = {Lee, Michael J. and DiCarlo, James J.}, editor = {Kietzmann, Tim Christian} } @article {184, title = {How well do rudimentary plasticity rules predict adult visual object learning?}, journal = {PLOS Computational Biology}, volume = {19}, year = {2023}, month = {Nov-12-2023}, pages = {e1011713}, abstract = {

A core problem in visual object learning is using a finite number of images of a new object to accurately identify that object in future, novel images.

One longstanding, conceptual hypothesis asserts that this core problem is solved by adult brains through two connected mechanisms: 1) the re-representation of incoming retinal images as points in a fixed, multidimensional neural space, and 2) the optimization of linear decision boundaries in that space, via simple plasticity rules applied to a single downstream layer.

Though this scheme is biologically plausible, the extent to which it explains learning behavior in humans has been unclear\—in part because of a historical lack of image-computable models of the putative neural space, and in part because of a lack of measurements of human learning behaviors in difficult, naturalistic settings.

Here, we addressed these gaps by 1) drawing from contemporary, image-computable models of the primate ventral visual stream to create a large set of testable learning models (n = 2,408 models), and 2) using online psychophysics to measure human learning trajectories over a varied set of tasks involving novel 3D objects (n = 371,000 trials), which we then used to develop (and\ publicly release) empirical benchmarks for comparing learning models to humans.

We evaluated each learning model on these benchmarks, and found those based on deep, high-level representations from neural networks were surprisingly aligned with human behavior. While no tested model explained the entirety of replicable human behavior, these results establish that rudimentary plasticity rules, when combined with appropriate visual representations, have high explanatory power in predicting human behavior with respect to this core object learning problem.

}, issn = {1553-734X}, doi = {10.1371/journal.pcbi.1011713}, url = {https://dx.plos.org/10.1371/journal.pcbi.1011713}, author = {Lee, Michael J. and DiCarlo, James J.}, editor = {Kietzmann, Tim Christian} } @conference {182, title = {Probing Biological and Artificial Neural Networks with Task-dependent Neural Manifolds}, booktitle = {Conference on Parsimony and Learning (Proceedings Track)}, year = {2023}, address = {Hong Kong, China}, abstract = {

Recently, growth in our understanding of the computations performed in both biological and artificial neural networks has largely been driven by either low-level mechanistic studies or global normative approaches. However, concrete methodologies for bridging the gap between these levels of abstraction remain elusive. In this work, we investigate the internal mechanisms of neural networks through the lens of neural population geometry, aiming to provide understanding at an intermediate level of abstraction, as a way to bridge that gap. Utilizing manifold capacity theory (MCT) from statistical physics and manifold alignment analysis (MAA) from high-dimensional statistics, we probe the underlying organization of task-dependent manifolds in deep neural networks and macaque neural recordings. Specifically, we quantitatively characterize how different learning objectives lead to differences in the organizational strategies of these models and demonstrate how these geometric analyses are connected to the decodability of task-relevant information. These analyses present a strong direction for bridging mechanistic and normative theories in neural networks through neural population geometry, potentially opening up many future research avenues in both machine learning and neuroscience.

}, keywords = {Biologically inspired vision models, computational neuroscience, Neural Geometry, Neural Manifolds, Neuro-AI, Representational Geometry}, url = {https://openreview.net/forum?id=MxBS6aw5Gd}, author = {Michael Kuoch and Chi-Ning Chou and Nikhil Parthasarathy and Joel Dapello and James J. DiCarlo and Haim Sompolinsky and Sueyeon Chung} } @article {181, title = {The Quest for an Integrated Set of Neural Mechanisms Underlying Object Recognition in Primates}, journal = {arXiv}, year = {2023}, month = {12/10/2023}, type = {preprint}, abstract = {

Visual object recognition -- the behavioral ability to rapidly and accurately categorize many visually encountered objects -- is core to primate cognition. This behavioral capability is algorithmically impressive because of the myriad identity-preserving viewpoints and scenes that dramatically change the visual image produced by the same object. Until recently, the brain mechanisms that support that capability were deeply mysterious. However, over the last decade, this scientific mystery has been illuminated by the discovery and development of brain-inspired, image-computable, artificial neural network (ANN) systems that rival primates in this behavioral feat. Apart from fundamentally changing the landscape of artificial intelligence (AI), modified versions of these ANN systems are the current leading scientific hypotheses of an integrated set of mechanisms in the primate ventral visual stream that support object recognition. What separates brain-mapped versions of these systems from prior conceptual models is that they are Sensory-computable, Mechanistic, Anatomically Referenced, and Testable (SMART). Here, we review and provide perspective on the brain mechanisms that the currently leading SMART models address. We review the empirical brain and behavioral alignment successes and failures of those current models. Given ongoing advances in neurobehavioral measurements and AI, we discuss the next frontiers for even more accurate mechanistic understanding. And we outline the likely applications of that SMART-model-based understanding.

}, doi = { https://doi.org/10.48550/arXiv.2312.05956}, url = {https://arxiv.org/pdf/2312.05956.pdf}, author = {Kohitij Kar and James J DiCarlo} } @article {171, title = {Aligning Model and Macaque Inferior Temporal Cortex Representations Improves Model-to-Human Behavioral Alignment and Adversarial Robustness}, journal = {bioRxiv}, year = {2022}, month = {July 4, 20202}, type = {preprint}, abstract = {

While some state-of-the-art artificial neural network systems in computer vision are strikingly accurate models of the corresponding primate visual processing, there are still many discrepancies between these models and the behavior of primates on object recognition tasks. Many current models suffer from extreme sensitivity to adversarial attacks and often do not align well with the image-by-image behavioral error patterns observed in humans. Previous research has provided strong evidence that primate object recognition behavior can be very accurately predicted by neural population activity in the inferior temporal (IT) cortex, a brain area in the late stages of the visual processing hierarchy. Therefore, here we directly test whether making the late stage representations of models more similar to that of macaque IT produces new models that exhibit more robust, primate-like behavior. We conducted chronic, large-scale multi-electrode recordings across the IT cortex in six non-human primates (rhesus macaques). We then use these data to fine-tune (end-to-end) the model \“IT\” representations such that they are more aligned with the biological IT representations, while preserving accuracy on object recognition tasks. We generate a cohort of models with a range of IT similarity scores validated on held-out animals across two image sets with distinct statistics. Across a battery of optimization conditions, we observed a strong correlation between the models\’ IT-likeness and alignment with human behavior, as well as an increase in its adversarial robustness. We further assessed the limitations of this approach and find that the improvements in behavioral alignment and adversarial robustness generalize across different image statistics, but not to object categories outside of those covered in our IT training set. Taken together, our results demonstrate that building models that are more aligned with the primate brain leads to more robust and human-like behavior, and call for larger neural data-sets to further augment these gains.Competing Interest StatementThe authors have declared no competing interest.

}, doi = {https://doi.org/10.1101/2022.07.01.498495}, url = {https://www.biorxiv.org/content/10.1101/2022.07.01.498495v1.full.pdf}, author = {Dapello, Joel and Kar, Kohitij and Schrimpf, Martin and Geary, Robert and Ferguson, Michael and Cox, David D. and DiCarlo, James J.} } @article {165, title = {Chemogenetic suppression of macaque V4 neurons produces retinotopically specific deficits in downstream IT neural activity patterns and core object recognition behavior}, journal = {Journal of Vision}, volume = {21}, year = {2021}, pages = {2489-2489}, abstract = {

Distributed activity patterns across multiple brain areas (e.g., V4, IT) enable primates to accurately identify visual objects. To strengthen our inferences about the causal role of underlying brain circuits, it is necessary to develop targeted neural perturbation strategies that enable discrimination amongst competing models. To probe the role of area V4 in core object recognition, we expressed inhibitory DREADDs in neurons within a 5x5 mm subregion of V4 cortex via multiple viral injections (AAV8-hSyn-hM4Di-mCherry; two macaques). To assay for successful neural suppression, we recorded from a multi-electrode array implanted over the transfected V4. We also recorded from multi-electrode arrays in the IT cortex (the primary feedforward target of V4), while simultaneously measuring the monkeys\’ behavior during object discrimination tasks. We found that systemic (intramuscular) injection of the DREADDs activator (CNO) produced reversible reductions (~20\%) in image-evoked V4 responses compared to the control condition (saline injections). Monkeys showed significant behavioral performance deficits upon CNO injections (compared to saline), which were larger when the object position overlapped with the RF estimates of the transfected V4 neurons. This is consistent with the hypothesis that the suppressed V4 neurons are critical to this behavior. Furthermore, we observed commensurate deficits in the linearly-decoded estimates of object identity from the IT population activity (post-CNO). To model the perturbed brain circuitry, we used a primate brain-mapped artificial neural network (ANN) model (CORnet-S) that supports object recognition. We \“lesioned\” the model\’s corresponding V4 subregion by modifying its weights such that the responses matched a subset of our experimental V4 measurements (post-CNO). Indeed, the lesioned model better predicted the measured (held-out) V4 and IT responses (post-CNO), compared to the model\&$\#$39;s non-lesioned version, validating our approach. In the future, our approach allows us to discriminate amongst competing mechanistic brain models, while the data provides constraints to guide more accurate alternatives.

}, doi = {https://doi.org/10.1167/jov.21.9.2489}, author = {Kar, Kohitij and Schrimpf, Martin and Schmidt, Kailyn and DiCarlo, JJ} } @article {161, title = {Computational models of category-selective brain regions enable high-throughput tests of selectivity}, journal = {Nature Communications}, volume = {12}, year = {2021}, month = {Jan-12-2021}, abstract = {

Cortical regions apparently selective to faces, places, and bodies have provided important evidence for domain-specific theories of human cognition, development, and evolution. But claims of category selectivity are not quantitatively precise and remain vulnerable to empirical refutation. Here we develop artificial neural network-based encoding models that accurately predict the response to novel images in the fusiform face area, parahippocampal place area, and extrastriate body area, outperforming descriptive models and experts. We use these models to subject claims of category selectivity to strong tests, by screening for and synthesizing images predicted to produce high responses. We find that these high-responsepredicted images are all unambiguous members of the hypothesized preferred category for each region. These results provide accurate, image-computable encoding models of each category-selective region, strengthen evidence for domain specificity in the brain, and point the way for future research characterizing the functional organization of the brain with unprecedented computational precision.

}, doi = {10.1038/s41467-021-25409-6}, url = {https://www.nature.com/articles/s41467-021-25409-6}, author = {Murty, NAR and Bashivan, Pouya and Abate, Alex and DiCarlo, James J. and Kanwisher, Nancy} } @article {153, title = {Fast Recurrent Processing Via Ventral Prefrontal Cortex is Needed by the Primate Ventral Stream for Robust Core Visual Object Recognition}, journal = {Neuron}, volume = {109}, year = {2021}, pages = {164-167.e5}, chapter = {164}, abstract = {

Distributed neural population spiking patterns in macaque inferior temporal (IT) cortex that support core visual object recognition require additional time to develop for specific (\“late-solved\”) images suggesting the necessity of recurrent processing in these computations. Which brain circuit motifs are most responsible for computing and transmitting these putative recurrent signals to IT? To test whether the ventral prefrontal cortex (vPFC) is a critical recurrent circuit node in this system, here we pharmacologically inactivated parts of the vPFC and simultaneously measured IT population activity, while monkeys performed object discrimination tasks. Our results show that vPFC inactivation deteriorated the quality of the late-phase (\>150 ms from image onset) IT population code, along with commensurate, specific behavioral deficits for \“late-solved\” images. Finally, silencing vPFC caused the monkeys\’ IT activity patterns and behavior to become more like those produced by feedforward artificial neural network models of the ventral stream. Together with prior work, these results argue that fast recurrent processing through the vPFC is critical to the production of behaviorally-sufficient object representations in IT.

}, doi = {https://doi.org/10.1016/j.neuron.2020.09.035}, url = {https://www.cell.com/neuron/pdf/S0896-6273(20)30759-5.pdf}, author = {Kar, Kohitij and DiCarlo, James J.} } @article {6, title = {Fast recurrent processing via ventral prefrontal cortex is needed by the primate ventral stream for robust core visual object recognition}, journal = {BioRxiv}, year = {2020}, month = {05/2020}, type = {preprint}, abstract = {

Distributed neural population spiking patterns in macaque inferior temporal (IT) cortex that support core visual object recognition require additional time to develop for specific (\"late-solved\") images suggesting the necessity of recurrent processing in these computations. Which brain circuit motifs are most responsible for computing and transmitting these putative recurrent signals to IT? To test whether the ventral prefrontal cortex (vPFC) is a critical recurrent circuit node in this system, here we pharmacologically inactivated parts of the vPFC and simultaneously measured IT population activity, while monkeys performed object discrimination tasks. Our results show that vPFC inactivation deteriorated the quality of the late-phase (\>150 ms from image onset) IT population code, along with commensurate, specific behavioral deficits for \"late-solved\" images. Finally, silencing vPFC caused the monkeys\&$\#$39; IT activity patterns and behavior to become more like those produced by feedforward artificial neural network models of the ventral stream. Together with prior work, these results argue that fast recurrent processing through the vPFC is critical to the production of behaviorally-sufficient object representations in IT.

}, doi = {https://doi.org/10.1101/2020.05.10.086959}, url = {https://www.biorxiv.org/content/10.1101/2020.05.10.086959v1}, author = {Kohitij Kar and James J. DiCarlo} } @article {141, title = {The inferior temporal cortex is a potential cortical precursor of orthographic processing in untrained monkeys}, journal = {Nature Communications}, volume = {11}, year = {2020}, month = {Jan-12-2020}, abstract = {

The ability to recognize written letter strings is foundational to human reading, but the underlying neuronal mechanisms remain largely unknown. Recent behavioral research in baboons suggests that non-human primates may provide an opportunity to investigate this question. We recorded the activity of hundreds of neurons in V4 and the inferior temporal cortex (IT) while na\ïve macaque monkeys passively viewed images of letters, English words and non-word strings, and tested the capacity of those neuronal representations to support a battery of orthographic processing tasks. We found that simple linear read-outs of IT (but not V4) population responses achieved high performance on all tested tasks, even matching the performance and error patterns of baboons on word classification. These results show that the IT cortex of untrained primates can serve as a precursor of orthographic processing, suggesting that the acquisition of reading in humans relies on the recycling of a brain network evolved for other visual functions.

}, doi = {10.1038/s41467-020-17714-3}, url = {http://www.nature.com/articles/s41467-020-17714-3}, author = {Rajalingham, Rishi and Kar, Kohitij and Sanghavi, Sachi and Dehaene, Stanislas and DiCarlo, James J.} } @article {147, title = {Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence}, journal = {Neuron}, year = {2020}, month = {Jan-09-2020}, abstract = {

A potentially organizing goal of the brain and cognitive sciences is to accurately explain domains of human intelligence as executable, neurally mechanistic models. Years of research have led to models that capture experimental results in individual behavioral tasks and individual brain regions. We here advocate for taking the next step: integrating experimental results from many laboratories into suites of benchmarks that, when considered together, push mechanistic models toward explaining entire domains of intelligence, such as vision, language, and motor control. Given recent successes of neurally mechanistic models and the surging availability of neural, anatomical, and behavioral data, we believe that now is the time to create integrative benchmarking platforms that incentivize ambitious, unified models. This perspective discusses the advantages and the challenges of this approach and proposes specific steps to achieve this goal in the domain of visual intelligence with the case study of an integrative benchmarking platform called Brain-Score.

}, issn = {08966273}, doi = {10.1016/j.neuron.2020.07.040}, url = {https://linkinghub.elsevier.com/retrieve/pii/S089662732030605X}, author = {Schrimpf, Martin and Kubilius, Jonas and Lee, Michael J. and Murty, NAR and Ajemian, Robert and DiCarlo, James J.} } @article {152, title = {An Open Resource for Non-human Primate Optogenetics}, journal = {Neuron}, year = {2020}, month = {January 10, 2020}, type = {NeuroResource}, abstract = {

Optogenetics has revolutionized neuroscience in small laboratory animals, but its effect on animal models more closely related to humans, such as non-human primates (NHPs), has been mixed. To make evidence-based decisions in primate optogenetics, the scientific community would benefit from a centralized database listing all attempts, successful and unsuccessful, of using optogenetics in the primate brain. We contacted members of the community to ask for their contributions to an open science initiative. As of this writing, 45 laboratories around the world contributed more than 1,000 injection experiments, including precise details regarding their methods and outcomes. Of those entries, more than half had not been published. The resource is free for everyone to consult and contribute to on the Open Science Framework website. Here we review some of the insights from this initial release of the database and discuss methodological considerations to improve the success of optogenetic experiments in NHPs.

}, issn = {08966273}, doi = {10.1016/j.neuron.2020.09.027}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0896627320307510}, author = {Tremblay, Sebastien and Acker, Leah and Afraz, Arash and Albaugh, Daniel L. and Amita, Hidetoshi and Andrei, Ariana R. and Angelucci, Alessandra and Aschner, Amir and Balan, Puiu F. and Basso, Michele A. and Benvenuti, Giacomo and Bohlen, Martin O. and Caiola, Michael J. and Calcedo, Roberto and Cavanaugh, James and Chen, Yuzhi and Chen, Spencer and Chernov, Mykyta M. and Clark, Andrew M. and Dai, Ji and Debes, Samantha R. and Deisseroth, Karl and Desimone, Robert and Dragoi, Valentin and Egger, Seth W. and Eldridge, Mark A.G. and El-Nahal, Hala G. and Fabbrini, Francesco and Federer, Frederick and Fetsch, Christopher R. and Fortuna, Michal G. and Friedman, Robert M. and Fujii, Naotaka and Gail, Alexander and Galvan, Adriana and Ghosh, Supriya and Gieselmann, Marc Alwin and Gulli, Roberto A. and Hikosaka, Okihide and Hosseini, Eghbal A. and Hu, Xing and {\"u}er, Janina and Inoue, Ken-ichi and Janz, Roger and Jazayeri, Mehrdad and Jiang, Rundong and Ju, Niansheng and Kar, Kohitij and Klein, Carsten and Kohn, Adam and Komatsu, Misako and Maeda, Kazutaka and Martinez-Trujillo, Julio C. and Matsumoto, Masayuki and Maunsell, John H.R. and Mendoza-Halliday, Diego and Monosov, Ilya E. and Muers, Ross S. and Nurminen, Lauri and Ortiz-Rios, Michael and {\textquoteright}Shea, Daniel J. and Palfi, {\'e}phane and Petkov, Christopher I. and Pojoga, Sorin and Rajalingham, Rishi and Ramakrishnan, Charu and Remington, Evan D. and Revsine, Cambria and Roe, Anna W. and Sabes, Philip N. and Saunders, Richard C. and Scherberger, {\"o}rg and Schmid, Michael C. and Schultz, Wolfram and Seidemann, Eyal and Senova, Yann-Suhan and Shadlen, Michael N. and Sheinberg, David L. and Siu, Caitlin and Smith, Yoland and Solomon, Selina S. and Sommer, Marc A. and Spudich, John L. and Stauffer, William R. and Takada, Masahiko and Tang, Shiming and Thiele, Alexander and Treue, Stefan and Vanduffel, Wim and Vogels, Rufin and Whitmire, Matthew P. and Wichmann, Thomas and Wurtz, Robert H. and Xu, Haoran and Yazdan-Shahmorad, Azadeh and Shenoy, Krishna V. and DiCarlo, James J. and Platt, Michael L.} } @article {148, title = {ThreeDWorld: A Platform for Interactive Multi-Modal Physical Simulation}, journal = {arXiv}, year = {2020}, month = {July 9, 2020}, type = {preprint}, abstract = {

We introduce ThreeDWorld (TDW), a platform for interactive multi-modal physical simulation. With TDW, users can simulate high-fidelity sensory data and physical interactions between mobile agents and objects in a wide variety of rich 3D environments. TDW has several unique properties: 1) realtime near photo-realistic image rendering quality; 2) a library of objects and environments with materials for high-quality rendering, and routines enabling user customization of the asset library; 3) generative procedures for efficiently building classes of new environments 4) high-fidelity audio rendering; 5) believable and realistic physical interactions for a wide variety of material types, including cloths, liquid, and deformable objects; 6) a range of \"avatar\" types that serve as embodiments of AI agents, with the option for user avatar customization; and 7) support for human interactions with VR devices. TDW also provides a rich API enabling multiple agents to interact within a simulation and return a range of sensor and physics data representing the state of the world. We present initial experiments enabled by the platform around emerging research directions in computer vision, machine learning, and cognitive science, including multi-modal physical scene understanding, multi-agent interactions, models that \"learn like a child\", and attention studies in humans and neural networks. The simulation platform will be made publicly available.

}, url = {https://arxiv.org/abs/2007.04954}, author = {Gan, Chuang and Schwartz, Jeremy and Alter, Seth and Schrimpf, Martin and Traer, James and De Freitas, Julian and Kubilius, Jonas and Bhandwaldar, Abhishek and Haber, Nick and Sano, Megumi and Wang, Elias and Mrowca, Damian and Lingelbach, Michael and Curtis, Aidan and Figelis, Kevin and Bear, Daniel M. and Gutfreund, Dan and Cox, David and DiCarlo, James J. and McDermott, Josh and Tenenbaum, Joshua B. and Yamins, Daniel L.K.} } @article {5, title = {Topographic deep artificial neural networks reproduce the hallmarks of the primate inferior temporal cortex face processing network}, journal = {bioRxiv}, year = {2020}, month = {07/2020}, type = {preprint}, abstract = {

A salient characteristic of monkey inferior temporal (IT) cortex is the IT face processing network. Its hallmarks include: \“face neurons\” that respond more to faces than non-face objects, strong spatial clustering of those neurons in foci at each IT anatomical level (\“face patches\”), and the preferential interconnection of those foci. While some deep artificial neural networks (ANNs) are good predictors of IT neuronal responses, including face neurons, they do not explain those face network hallmarks. Here we ask if they might be explained with a simple, metabolically motivated addition to current ANN ventral stream models. Specifically, we designed and successfully trained topographic deep ANNs (TDANNs) to solve real-world visual recognition tasks (as in prior work), but, in addition, we also optimized each network to minimize a proxy for neuronal wiring length within its IT layers. We report that after this dual optimization, the model IT layers of TDANNs reproduce the hallmarks of the IT face network: the presence of face neurons, clusters of face neurons that quantitatively match those found in IT face patches, connectivity between those patches, and the emergence of face viewpoint invariance along the network hierarchy. We find that these phenomena emerge for a range of naturalistic experience, but not for highly unnatural training. Taken together, these results show that the IT face processing network could be a consequence of a basic hierarchical anatomy along the ventral stream, selection pressure on the visual system to accomplish general object categorization, and selection pressure to minimize axonal wiring length.

}, doi = {https://doi.org/10.1101/2020.07.09.185116}, url = {https://www.biorxiv.org/content/10.1101/2020.07.09.185116v1.full.pdf}, author = {Hyodong Lee and Eshed Margalit and Kamila M. Jozwik and Michael A. Cohen and Nancy Kanwisher and Daniel L. K. Yamins and James J. DiCarlo} } @conference {11, title = {Brain-Like Object Recognition with High-Performing Shallow Recurrent ANNs}, booktitle = {Neural Information Processing Systems}, year = {2019}, abstract = {

Deep convolutional artificial neural networks (ANNs) are the leading class of candidate models of the mechanisms of visual processing in the primate ventral stream. While initially inspired by brain anatomy, over the past years, these ANNs have evolved from a simple eight-layer architecture in AlexNet to extremely deep and branching architectures, demonstrating increasingly better object categorization performance, yet bringing into question how brain-like they still are. In particular, typical deep models from the machine learning community are often hard to map onto the brain\&$\#$39;s anatomy due to their vast number of layers and missing biologically-important connections, such as recurrence. Here we demonstrate that better anatomical alignment to the brain and high performance on machine learning as well as neuroscience measures do not have to be in contradiction. We developed CORnet-S, a shallow ANN with four anatomically mapped areas and recurrent connectivity, guided by Brain-Score, a new large-scale composite of neural and behavioral benchmarks for quantifying the functional fidelity of models of the primate ventral visual stream. Despite being significantly shallower than most models, CORnet-S is the top model on Brain-Score and outperforms similarly compact models on ImageNet. Moreover, our extensive analyses of CORnet-S circuitry variants reveal that recurrence is the main predictive factor of both Brain-Score and ImageNet top-1 performance. Finally, we report that the temporal evolution of the CORnet-S \"IT\" neural population resembles the actual monkey IT population dynamics. Taken together, these results establish CORnet-S, a compact, recurrent ANN, as the current best model of the primate ventral visual stream.

}, doi = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns}, url = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns.pdf}, author = {Jonas Kubilius and Martin Schrimpf and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kohitij Kar and Bashivan, Pouya and Jonathan Prescott-Roy and Kailyn Schmidt and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and James J. DiCarlo} } @article {4, title = {Evidence that recurrent circuits are critical to the ventral stream{\textquoteright}s execution of core object recognition behavior}, journal = {Nature Neuroscience}, volume = {22}, year = {2019}, month = {01/2019}, pages = {974 - 983}, issn = {1097-6256}, doi = {10.1038/s41593-019-0392-5}, url = {http://www.nature.com/articles/s41593-019-0392-5}, author = {Kar, Kohitij and Kubilius, Jonas and Schmidt, Kailyn and Issa, Elias B. and DiCarlo, James J.} } @inbook {12, title = {Funcitional Properties of Circuits, Cellular Populations, and Areas}, booktitle = {The Neocortex}, volume = {27}, year = {2019}, pages = {223-265}, publisher = {The MIT Press}, organization = {The MIT Press}, chapter = {13}, address = {Cambridge, MA}, issn = {978-0-262-04324-3}, doi = {10.7551/mitpress/12593.001.0001}, url = {http://www.esforum.de}, author = {Harris, KD and Groh, JM and DiCarlo, JJ and Fries, P and Kaschube, M and Laurent, G and MacLean, JN and McCormick, DA and Pipa, G and Reynolds, JN and Schwartz, AB and Sejnowksi, TJ and Singer, W and Vinck, M}, editor = {Singer, Wolf and Sejnowski, Terrence J. and Rakic, Pasko} } @article {8, title = {Neural population control via deep image synthesis}, journal = {Science}, volume = {364}, year = {2019}, month = {03/2019}, pages = {eaav9436}, abstract = {

Particular deep artificial neural networks (ANNs) are today\&$\#$39;s most accurate models of the primate brain\&$\#$39;s ventral visual stream. Using an ANN-driven image synthesis method, we found that luminous power patterns (i.e., images) can be applied to primate retinae to predictably push the spiking activity of targeted V4 neural sites beyond naturally occurring levels. This method, although not yet perfect, achieves unprecedented independent control of the activity state of entire populations of V4 neural sites, even those with overlapping receptive fields. These results show how the knowledge embedded in today\&$\#$39;s ANN models might be used to noninvasively set desired internal brain states at neuron-level resolution, and suggest that more accurate ANN models would produce even more accurate control.

}, issn = {0036-8075}, doi = {10.1126/science.aav9436}, url = {http://science.sciencemag.org/cgi/rapidpdf/364/6439/eaav9436?ijkey=iBRdlniG7iYuA\&keytype=ref\&siteid=sci}, author = {Bashivan, Pouya and Kar, Kohitij and DiCarlo, James J.} } @article {9, title = {To find better neural network models of human vision, find better neural network models of primate vision}, journal = {BioRxiv}, year = {2019}, month = {07/2019}, type = {preprint}, abstract = {

Specific deep artificial neural networks (ANNs) are the current best models of ventral visual processing and object recognition behavior in monkeys. We here explore whether models of non-human primate vision generalize to visual processing in the human primate brain. Specifically, we asked if model match to monkey IT is a predictor of model match to human IT, even when scoring those matches on different images. We found that the model match to monkey IT is a positive predictor of the model match to human IT (R = 0.36), and that this approach outperforms the current standard predictor of model accuracy on ImageNet. This suggests a more powerful approach for pre-selecting models as hypotheses of human brain processing.

}, doi = {https://doi.org/10.1101/688390}, url = {https://www.biorxiv.org/content/10.1101/688390v1.full.pdf}, author = {Kamila M. Jozwik and Martin Schrimpf and Nancy Kanwisher and James J. DiCarlo} } @conference {166, title = {Using Brain-Score to Evaluate and Build Neural Networks for Brain-Like Object Recognition}, booktitle = {Computational and Systems Neuroscience (COSYNE)}, year = {2019}, address = {Denver, CO}, author = {Schrimpf, Martin and Kubilius, Jonas and Hong, Ha and Majaj, Najib and Rajalingham, Rishi and Issa, Elias B and Kar, Kohitij and Ziemba, Corey M and Bashivan, Pouya and Prescott-Roy, Jonathan and Schmidt, Kailyn and Yamins, Daniel LK and DiCarlo, James J} } @article {22, title = {Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?}, journal = {bioRxiv}, year = {2018}, month = {09/2018}, type = {preprint}, abstract = {

The internal representations of early deep artificial neural networks (ANNs) were found to be remarkably similar to the internal neural representations measured experimentally in the primate brain. Here we ask, as deep ANNs have continued to evolve, are they becoming more or less brain-like? ANNs that are most functionally similar to the brain will contain mechanisms that are most like those used by the brain. We therefore developed Brain-Score - a composite of multiple neural and behavioral benchmarks that score any ANN on how similar it is to the brain\&$\#$39;s mechanisms for core object recognition - and we deployed it to evaluate a wide range of state-of-the-art deep ANNs. Using this scoring system, we here report that: (1) DenseNet-169, CORnet-S and ResNet-101 are the most brain-like ANNs. (2) There remains considerable variability in neural and behavioral responses that is not predicted by any ANN, suggesting that no ANN model has yet captured all the relevant mechanisms. (3) Extending prior work, we found that gains in ANN ImageNet performance led to gains on Brain-Score. However, correlation weakened at \>= 70\% top-1 ImageNet performance, suggesting that additional guidance from neuroscience is needed to make further advances in capturing brain mechanisms. (4) We uncovered smaller (i.e. less complex) ANNs that are more brain-like than many of the best-performing ImageNet models, which suggests the opportunity to simplify ANNs to better understand the ventral stream. The scoring system used here is far from complete. However, we propose that evaluating and tracking model-benchmark correspondences through a Brain-Score that is regularly updated with new brain data is an exciting opportunity: experimental benchmarks can be used to guide machine network evolution, and machine networks are mechanistic hypotheses of the brain\&$\#$39;s network and thus drive next experiments. To facilitate both of these, we release Brain-Score.org: a platform that hosts the neural and behavioral benchmarks, where ANNs for visual processing can be submitted to receive a Brain-Score and their rank relative to other models, and where new experimental data can be naturally incorporated.

}, doi = {https://doi.org/10.1101/407007}, url = {https://www.biorxiv.org/content/10.1101/407007v2.full.pdf}, author = {Martin Schrimpf and Kubilius, Jonas and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kar, Kohitij and Bashivan, Pouya and Jonathan Prescott-Roy and Schmidt, Kailyn and Daniel L. K. Yamins and DiCarlo, James J.} } @article {23, title = {CORnet: Modeling the Neural Mechanisms of Core Object Recognition}, journal = {bioRxiv}, year = {2018}, month = {09/2018}, type = {preprint}, abstract = {

Deep artificial neural networks with spatially repeated processing (a.k.a., deep convolutional ANNs) have been established as the best class of candidate models of visual processing in primate ventral visual processing stream. Over the past five years, these ANNs have evolved from a simple feedforward eight-layer architecture in AlexNet to extremely deep and branching NASNet architectures, demonstrating increasingly better object categorization performance and increasingly better explanatory power of both neural and behavioral responses. However, from the neuroscientist\&$\#$39;s point of view, the relationship between such very deep architectures and the ventral visual pathway is incomplete in at least two ways. On the one hand, current state-of-the-art ANNs appear to be too complex (e.g., now over 100 levels) compared with the relatively shallow cortical hierarchy (4-8 levels), which makes it difficult to map their elements to those in the ventral visual stream and to understand what they are doing. On the other hand, current state-of-the-art ANNs appear to be not complex enough in that they lack recurrent connections and the resulting neural response dynamics that are commonplace in the ventral visual stream. Here we describe our ongoing efforts to resolve both of these issues by developing a \"CORnet\" family of deep neural network architectures. Rather than just seeking high object recognition performance (as the state-of-the-art ANNs above), we instead try to reduce the model family to its most important elements and then gradually build new ANNs with recurrent and skip connections while monitoring both performance and the match between each new CORnet model and a large body of primate brain and behavioral data. We report here that our current best ANN model derived from this approach (CORnet-S) is among the top models on Brain-Score, a composite benchmark for comparing models to the brain, but is simpler than other deep ANNs in terms of the number of convolutions performed along the longest path of information processing in the model. All CORnet models are available at\ https://github.com/dicarlolab/CORnet, and we plan to update this manuscript and the available models in this family as they are produced.

}, doi = {https://doi.org/10.1101/408385}, url = {https://www.biorxiv.org/content/10.1101/408385v1.full.pdf}, author = {Kubilius, Jonas and Martin Schrimpf and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and DiCarlo, James J.} } @article {19, title = {Evidence that recurrent circuits are critical to the ventral stream{\textquoteright}s execution of core object recognition behavior}, journal = {bioRxiv}, year = {2018}, month = {06/2018}, type = {preprint}, abstract = {

Non-recurrent deep convolutional neural networks (DCNNs) are currently the best models of core object recognition; a behavior supported by the densely recurrent primate ventral stream, culminating in the inferior temporal (IT) cortex. Are these recurrent circuits critical to ventral stream\&$\#$39;s execution of this behavior? We reasoned that, if recurrence is critical, then primates should outperform feedforward-only DCNNs for some images, and that these images should require additional processing time beyond the feedforward IT response. Here we first used behavioral methods to discover hundreds of these \"challenge\" images. Second, using large-scale IT electrophysiology in animals performing core recognition tasks, we observed that behaviorally-sufficient, linearly-decodable object identity solutions emerged ~30ms (on average) later in IT for challenge images compared to DCNN and primate performance-matched \"control\" images. We observed these same late solutions even during passive viewing. Third, consistent with a failure of feedforward computations, the behaviorally-critical late-phase IT population response patterns evoked by the challenge images were poorly predicted by DCNN activations. Interestingly, deeper CNNs better predicted these late IT responses, suggesting a functional equivalence between recurrence and additional nonlinear transformations. Our results argue that automatically-evoked recurrent circuits are critical even for rapid object identification. By precisely comparing current DCNNs, primate behavior and IT population dynamics, we provide guidance for future recurrent model development.

}, doi = {https://doi.org/10.1101/354753}, url = {https://www.biorxiv.org/content/10.1101/354753v1.full.pdf}, author = {Kar, Kohitij and Kubilius, Jonas and Schmidt, Kailyn and Issa, Elias B and DiCarlo, James J.} } @article {13, title = {Large-Scale, High-Resolution Comparison of the Core Visual Object Recognition Behavior of Humans, Monkeys, and State-of-the-Art Deep Artificial Neural Networks}, journal = {The Journal of Neuroscience}, volume = {38}, year = {2018}, month = {03/2019}, pages = {7255 - 7269}, abstract = {

Primates-including humans-can typically recognize objects in visual images at a glance even in spite of naturally occurring identity-preserving image transformations (e.g. changes in viewpoint). A primary neuroscience goal is to uncover neuron-level mechanistic models that quantitatively explain this behavior by predicting primate performance for each and every image. Here, we applied this stringent behavioral prediction test to the leading mechanistic models of primate vision (specifically, deep, convolutional, artificial neural networks; ANNs) by directly comparing their behavioral signatures against those of humans and rhesus macaque monkeys. Using high-throughput data collection systems for human and monkey psychophysics, we collected over one million behavioral trials from 1472 anonymous humans and five male macaque monkeys for 2400 images over 276 binary object discrimination tasks. Consistent with previous work, we observed that state-of-the-art deep, feed-forward convolutional ANNs trained for visual categorization (termed DCNN models) accurately predicted primate patterns of object-level confusion. However, when we examined behavioral performance for individual images within each object discrimination task, we found that all tested DCNN models were significantly non-predictive of primate performance, and that this prediction failure was not accounted for by simple image attributes, nor rescued by simple model modifications. These results show that current DCNN models cannot account for the image-level behavioral patterns of primates, and that new ANN models are needed to more precisely capture the neural mechanisms underlying primate object vision. To this end, large-scale, high-resolution primate behavioral benchmarks-such as those obtained here-could serve as direct guides for discovering such models.Recently, specific feed-forward deep convolutional artificial neural networks (ANNs) models have dramatically advanced our quantitative understanding of the neural mechanisms underlying primate core object recognition. In this work, we tested the limits of those ANNs by systematically comparing the behavioral responses of these models with the behavioral responses of humans and monkeys, at the resolution of individual images. Using these high-resolution metrics, we found that all tested ANN models significantly diverged from primate behavior. Going forward, these high-resolution, large-scale primate behavioral benchmarks could serve as direct guides for discovering better ANN models of the primate visual system.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0388-18.2018}, url = {http://www.jneurosci.org/content/38/33/7255}, author = {Rajalingham, Rishi and Issa, Elias B. and Bashivan, Pouya and Kar, Kohitij and Schmidt, Kailyn and DiCarlo, James J.} } @article {21, title = {Large-scale, high-resolution comparison of the core visual object recognition behavior of humans, monkeys, and state-of-the-art deep artificial neural networks}, journal = {bioRxiv}, year = {2018}, month = {02/2018}, type = {preprint}, abstract = {

Primates\—including humans\—can typically recognize objects in visual images at a glance even in the face of naturally occurring identity-preserving image transformations (e.g. changes in viewpoint). A primary neuroscience goal is to uncover neuron-level mechanistic models that quantitatively explain this behavior by predicting primate performance for each and every image. Here, we applied this stringent behavioral prediction test to the leading mechanistic models of primate vision (specifically, deep, convolutional, artificial neural networks; ANNs) by directly comparing their behavioral signatures against those of humans and rhesus macaque monkeys. Using high-throughput data collection systems for human and monkey psychophysics, we collected over one million behavioral trials for 2400 images over 276 binary object discrimination tasks. Consistent with previous work, we observed that state-of-the-art deep, feed-forward convolutional ANNs trained for visual categorization (termed DCNNIC\ models) accurately predicted primate patterns of object-level confusion. However, when we examined behavioral performance for individual images within each object discrimination task, we found that all tested DCNNIC\ models were significantly non-predictive of primate performance, and that this prediction failure was not accounted for by simple image attributes, nor rescued by simple model modifications. These results show that current DCNNIC\ models cannot account for the image-level behavioral patterns of primates, and that new ANN models are needed to more precisely capture the neural mechanisms underlying primate object vision. To this end, large-scale, high-resolution primate behavioral benchmarks\—such as those obtained here\—could serve as direct guides for discovering such models.

}, doi = {https://doi.org/10.1101/240614}, url = {https://www.biorxiv.org/content/10.1101/240614v4.full.pdf}, author = {Rajalingham, Rishi and Issa, Elias B and Bashivan, Pouya and Kar, Kohitij and Schmidt, Kailyn and DiCarlo, James J.} } @article {20, title = {Task-Driven Convolutional Recurrent Models of the Visual System}, journal = {arXiv}, year = {2018}, month = {06/2018}, type = {preprint}, abstract = {

Feed-forward convolutional neural networks (CNNs) are currently state-of-the-art for object classification tasks such as ImageNet. Further, they are quantitatively accurate models of temporally-averaged responses of neurons in the primate brain\&$\#$39;s visual system. However, biological visual systems have two ubiquitous architectural features not shared with typical CNNs: local recurrence within cortical areas, and long-range feedback from downstream areas to upstream areas. Here we explored the role of recurrence in improving classification performance. We found that standard forms of recurrence (vanilla RNNs and LSTMs) do not perform well within deep CNNs on the ImageNet task. In contrast, custom cells that incorporated two structural features, bypassing and gating, were able to boost task accuracy substantially. We extended these design principles in an automated search over thousands of model architectures, which identified novel local recurrent cells and long-range feedback connections useful for object recognition. Moreover, these task-optimized ConvRNNs explained the dynamics of neural activity in the primate visual system better than feedforward networks, suggesting a role for the brain\&$\#$39;s recurrent connections in performing difficult visual behaviors.

}, doi = {https://arxiv.org/abs/1807.00053}, url = {https://arxiv.org/pdf/1807.00053.pdf}, author = {Aran Nayebi and Daniel Bear and Kubilius, Jonas and Ganguli, S and Sussillo, D and DiCarlo, James J. and Yamins, DLK} } @article {75, title = {A Stable Topography of Selectivity for Unfamiliar Shape Classes in Monkey Inferior Temporal Cortex}, journal = {Cerebral Cortex}, volume = {18}, year = {2007}, month = {2007}, pages = {1676 - 1694}, abstract = {

The inferior temporal {(IT)} cortex in monkeys plays a central role in visual object recognition and learning. Previous studies have observed patches in {IT} cortex with strong selectivity for highly familiar object classes (e.g., faces), but the principles behind this functional organization are largely unknown due to the many properties that distinguish different object classes. To unconfound shape from meaning and memory, we scanned monkeys with functional magnetic resonance imaging while they viewed classes of initially novel objects. Our data revealed a topography of selectivity for these novel object classes across {IT} cortex. We found that this selectivity topography was highly reproducible and remarkably stable across a 3-month interval during which monkeys were extensively trained to discriminate among exemplars within one of the object classes. Furthermore, this selectivity topography was largely unaffected by changes in behavioral task and object retinal position, both of which preserve shape. In contrast, it was strongly influenced by changes in object shape. The topography was partially related to, but not explained by, the previously described pattern of face selectivity. Together, these results suggest that {IT} cortex contains a large-scale map of shape that is largely independent of meaning, familiarity, and behavioral task.

}, issn = {1047-3211}, doi = {10.1093/cercor/bhm196}, url = {https://academic.oup.com/cercor/article-lookup/doi/10.1093/cercor/bhm196}, author = {Op de Beeck, Hans P. and Deutsch, Jennifer A. and Vanduffel, Wim and Kanwisher, Nancy G. and DiCarlo, James J.} } @article {78, title = {Trade-Off between Object Selectivity and Tolerance in Monkey Inferotemporal Cortex}, journal = {Journal of Neuroscience}, volume = {27}, year = {2007}, month = {07/2007}, pages = {12292 - 12307}, abstract = {

Object recognition requires both selectivity among different objects and tolerance to vastly different retinal images of the same object, resulting from natural variation in (e.g.) position, size, illumination, and clutter. Thus, discovering neuronal responses that have object selectivity and tolerance to identity-preserving transformations is fundamental to understanding object recognition. Although selectivity and tolerance are found at the highest level of the primate ventral visual stream [the inferotemporal cortex {(IT)],} both properties are highly varied and poorly understood. If an {IT} neuron has very sharp selectivity for a unique combination of object features (\"diagnostic features\"), this might automatically endow it with high tolerance. However, this relationship cannot be taken as given; although some {IT} neurons are highly object selective and some are highly tolerant, the empirical connection of these key properties is unknown. In this study, we systematically measured both object selectivity and tolerance to different identity-preserving image transformations in the spiking responses of a population of monkey {IT} neurons. We found that {IT} neurons with high object selectivity typically have low tolerance (and vice versa), regardless of how object selectivity was quantified and the type of tolerance examined. The discovery of this trade-off illuminates object selectivity and tolerance in {IT} and unifies a range of previous, seemingly disparate results. This finding also argues against the idea that diagnostic conjunctions of features guarantee tolerance. Instead, it is naturally explained by object recognition models in which object selectivity is built through {AND-like} tuning mechanisms.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.1897-07.2007}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.1897-07.2007}, author = {Zoccolan, D. and Kouh, M. and Poggio, T. and DiCarlo, J. J.} } @article {82, title = {Discrimination Training Alters Object Representations in Human Extrastriate Cortex}, journal = {Journal of Neuroscience}, volume = {26}, year = {2006}, month = {2006}, pages = {13025 - 13036}, abstract = {

Visual object recognition relies critically on learning. However, little is known about the effect of object learning in human visual cortex, and in particular how the spatial distribution of training effects relates to the distribution of object and face selectivity across the cortex before training. We scanned human subjects with high-resolution functional magnetic resonance imaging {(fMRI)} while they viewed novel object classes, both before and after extensive training to discriminate between exemplars within one of these object classes. Training increased the strength of the response in visual cortex to trained objects compared with untrained objects. However, training did not simply induce a uniform increase in the response to trained objects: the magnitude of this training effect varied substantially across subregions of extrastriate cortex, with some showing a twofold increase in response to trained objects and others (including the right fusiform face area) showing no significant effect of training. Furthermore, the spatial distribution of training effects could not be predicted from the spatial distribution of either pretrained responses or face selectivity. Instead, training changed the spatial distribution of activity across the cortex. These findings support a dynamic view of the ventral visual pathway in which the cortical representation of an object category is continuously modulated by experience.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.2481-06.2006}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.2481-06.2006}, author = {Op de Beeck, H. P. and Baker, C. I. and DiCarlo, J. J. and Kanwisher, N. G.} } @proceedings {84, title = {A large-scale shape map in monkey inferior temporal cortex}, journal = {Society for Neuroscience}, year = {2006}, month = {10/2006}, publisher = {SFN}, address = {Atlanta, GA, USA}, author = {Op de Beeck, H. P. and Deutsch, Jennifer A. and Vanduffel, Wim and Nancy Kanwisher and DiCarlo, James J.} } @article {83, title = {Learning and neural plasticity in visual object recognition}, journal = {Current Opinion in Neurobiology}, volume = {16}, year = {2006}, month = {01/2006}, pages = {152 - 158}, abstract = {

The capability of the adult primate visual system for rapid and accurate recognition of targets in cluttered, natural scenes far surpasses the abilities of state-of-the-art artificial vision systems. Understanding this capability remains a fundamental challenge in visual neuroscience. Recent experimental evidence suggests that adaptive coding strategies facilitated by underlying neural plasticity enable the adult brain to learn from visual experience and shape its ability to integrate and recognize coherent visual objects.

}, issn = {09594388}, doi = {10.1016/j.conb.2006.03.012}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0959438806000377}, author = {Kourtzi, Zoe and DiCarlo, James J} } @article {80, title = {Object Selectivity of Local Field Potentials and Spikes in the Macaque Inferior Temporal Cortex}, journal = {Neuron}, volume = {49}, year = {2006}, month = {01/2006}, pages = {433 - 445}, abstract = {

Local field potentials (LFPs) arise largely from dendritic activity over large brain regions and thus provide a measure of the input to and local processing within an area. We characterized (LFPs) and their relationship to spikes (multi and single unit) in monkey inferior temporal cortex (IT). LFP responses in IT to complex objects showed strong selectivity at 44\% of the sites and tolerance to retinal position and size. The LFP preferences were poorly predicted by the spike preferences at the same site but were better explained by averaging spikes within approximately 3 mm. A comparison of separate sites suggests that selectivity is similar on a scale of approximately 800 microm for spikes and approximately 5 mm for LFPs. These observations imply that inputs to IT neurons convey selectivity for complex shapes and that such input may have an underlying organization spanning several millimeters.

}, issn = {08966273}, doi = {10.1016/j.neuron.2005.12.019}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0896627305011311}, author = {Kreiman, Gabriel and Hung, Chou P. and Kraskov, Alexander and Quiroga, Rodrigo Quian and Poggio, Tomaso and DiCarlo, James J.} } @proceedings {86, title = {Trade-off between shape selectivity and tolerance to identity-preserving transformations in monkey inferotemporal cortex}, journal = {Gordon Conference: Sensation and the Natural Environment}, year = {2006}, month = {2006}, address = {Bozeman, MT, USA}, author = {Zoccolan, Davide and Kouh, M and Poggio, Tomaso and DiCarlo, James J.} } @article {88, title = {Fast Readout of Object Identity from Macaque Inferior Temporal Cortex}, journal = {Science}, volume = {310}, year = {2005}, month = {04/2005}, pages = {863 - 866}, abstract = {

Understanding the brain computations leading to object recognition requires quantitative characterization of the information represented in inferior temporal (IT) cortex. We used a biologically plausible, classifier-based readout technique to investigate the neural coding of selectivity and invariance at the IT population level. The activity of small neuronal populations (approximately 100 randomly selected cells) over very short time intervals (as small as 12.5 milliseconds) contained unexpectedly accurate and robust information about both object \"identity\" and \"category.\" This information generalized over a range of object positions and scales, even for novel objects. Coarse information about position and scale could also be read out from the same population.

}, keywords = {Action Potentials, Animals, Brain Mapping, Macaca mulatta, Neurons, Psychology, Psychomotor Performance, Recognition, Temporal Lobe, Time Factors, Visual Perception}, issn = {0036-8075}, doi = {10.1126/science.1117593}, url = {https://www.sciencemag.org/lookup/doi/10.1126/science.1117593}, author = {Hung, Chou P. and Kreiman, Gabriel and Poggio, Tomaso and DiCarlo, James J.} } @article {93, title = {Ultra-fast object recognition from few spikes}, number = {2005-022}, year = {2005}, month = {07/2005}, pages = {1-31}, institution = {MIT}, address = {Cambridge, MA}, abstract = {

Understanding the complex brain computations leading to object recognition requires quantitatively characterizing the information represented in inferior temporal cortex (IT), the highest stage of the primate visual stream. A read-out technique based on a trainable classifier is used to characterize the neural coding of selectivity and invariance at the population level. The activity of very small populations of independently recorded IT neurons (~100 randomly selected cells) over very short time intervals (as small as 12.5 ms) contains surprisingly accurate and robust information about both object \Â\‘identity\Â\’ and \Â\‘category\Â\’, which is furthermore highly invariant to object position and scale. Significantly, selectivity and invariance are present even for novel objects, indicating that these properties arise from the intrinsic circuitry and do not require object-specific learning. Within the limits of the technique, there is no detectable difference in the latency or temporal resolution of the IT information supporting so-called \Â\‘categorization\Â\’ (a.k. basic level) and \Â\‘identification\Â\’ (a.k. subordinate level) tasks. Furthermore, where information, in particular information about stimulus location and scale, can also be read-out from the same small population of IT neurons. These results show how it is possible to decode invariant object information rapidly, accurately and robustly from a small population in IT and provide insights into the nature of the neural code for different kinds of object-related information.

}, keywords = {AI, inferior temporal cortex, neural coding, object recognition}, issn = {2005-022}, url = {https://dspace.mit.edu/handle/1721.1/30556}, author = {Hung, Chou P. and Kreiman, Gabriel and Poggio, Tomaso and DiCarlo, James J.} } @proceedings {94, title = {Using {\textquoteleft}read-out{\textquoteright} of object identity to understand object coding in the macaque anterior inferior temporal cortex}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2005}, month = {03/2005}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, abstract = {

Recent efforts to develop robust computer vision systems capable of approaching the level of human object recognition performance have shown that models based on neurobiology can outperform non-biological models. To continue the development of such systems, we must understand the codes used by neuronal ensembles to represent object identity--a matter of continued debate in the neuroscience community. Understanding such codes would also allow the development of brain-machine interfaces capable of \‘reading-out\’ or \‘writing-in\’ information to AIT. One approach to understanding these neuronal codes is to obtain a large sample of neuronal data using a fixed set of objects, and then test the ability of various neuronal population measures (codes) to convey knowledge of object identity \– that is, determine the ability of each putative neuronal code to \‘read-out\’ object identity information from AIT. To this end, we have now recorded single-unit activity (SUA), multiunit activity (MUA) and local field potentials (LFP), from over 500 sites in the anterior inferior temporal cortex (AIT) of two macaque monkeys while they viewed a fixed set of 77 complex objects. In this paper, we focus on the temporal latency and resolution of AIT neuronal population codes that best convey knowledge of object identity as assessed by object classification. Specifically, the 77 objects were divided before the experiment into eight different classes: toys, foodstuffs, human faces, monkey faces, hands, vehicles, boxes and cats/dogs. To evaluate the performance of each putative neuronal population code, we used support vector machines (SVM) and jackknife cross-validation to avoid over-fitting. Under the assumption of no covariance across spatially separate sites, these data and methods allowed us to examine putative neuronal population codes that differed in their latency (0 to 100 ms from stimulus onset) and their temporal resolution (12.5 to 200 ms wide bins). Results were evaluated for consistency across different sets of recording sites. We found that MUA and SUA spike count codes could perform single trial object classification. Performance improved with the number of sites and reached ~90\% when ~128 arbitrary AIT sites were included. For any given number of sites, increasing the temporal resolution beyond ~50 ms did not lead to improved performance and the optimal latency from stimulus onset was ~100 ms. Because the neuronal data were not collected simultaneously, we cannot rule out spatio-temporal codes with a time scale less than ~50 ms. However, these results support the hypothesis that the time scale of the object identity code in AIT is ~50 ms and that downstream read-out of object identity for behavior or further processing may simply integrate over this time scale.\ 

}, author = {Hung, Chou P. and Kreiman, Gabriel and Quiroga, Rodrigo Quian and Kraskov, Alexander and Poggio, Tomaso and DiCarlo, James J.} } @proceedings {96, title = {Object recognition by selective spike and LFP data in macaque inferior temporal cortex}, journal = {Society for Neuroscience}, year = {2004}, month = {10/2004}, publisher = {SFN}, address = {San Diego, CA, USA}, author = {Kreiman, Gabriel and Hung, Chou P. and Poggio, Tomaso and DiCarlo, James J.} } @article {98, title = {Selectivity of local field potentials in macaque inferior temporal cortex}, number = {2004-20}, year = {2004}, month = {09/2004}, institution = {MIT}, address = {Cambridge, M}, abstract = {

While single neurons in inferior temporal (IT) cortex show differential responses to distinct complex stimuli, little is known about the responses of populations of neurons in IT. We recorded single electrode data, including multi-unit activity (MUA) and local field potentials (LFP), from 618 sites in the inferior temporal cortex of macaque monkeys while the animals passively viewed 78 different pictures of complex stimuli. The LFPs were obtained by low-pass filtering the extracellular electrophysiological signal with a corner frequency of 300 Hz. As reported previously, we observed that spike counts from MUA showed selectivity for some of the pictures. Strikingly, the LFP data, which is thought to constitute an average over large numbers of neurons, also showed significantly selective responses. The LFP responses were less selective than the MUA responses both in terms of the proportion of selective sites as well as in the selectivity of each site. We observed that there was only little overlap between the selectivity of MUA and LFP recordings from the same electrode. To assess the spatial organization of selective responses, we compared the selectivity of nearby sites recorded along the same penetration and sites recorded from different penetrations. We observed that MUA selectivity was correlated on spatial scales up to 800 \&$\#$61549;m while the LFP selectivity was correlated over a larger spatial extent, with significant correlations between sites separated by several mm. Our data support the idea that there is some topographical arrangement to the organization of selectivity in inferior temporal cortex and that this organization may be relevant for the representation of object identity in IT.

}, keywords = {AI, inferior temporal cortex, local field potentials, object recognition}, issn = {2004-020}, url = {https://dspace.mit.edu/handle/1721.1/30417}, author = {Kreiman, Gabriel and Hung, Chou P. and Poggio, Tomasso and DiCarlo, James J.} }