@article {174, title = {Catalyzing next-generation Artificial Intelligence through NeuroAI}, journal = {Nature Communications}, volume = {14}, year = {2023}, month = {Jan-12-2023}, pages = {1597}, abstract = {

Neuroscience has long been an essential driver of progress in artificial intelligence (AI). We propose that to accelerate progress in AI, we must invest in fundamental research in NeuroAI. A core component of this is the embodied Turing test, which challenges AI animal models to interact with the sensorimotor world at skill levels akin to their living counterparts. The embodied Turing test shifts the focus from those capabilities like game playing and language that are especially well-developed or uniquely human to those capabilities \– inherited from over 500 million years of evolution \– that are shared with all animals. Building models that can pass the embodied Turing test will provide a roadmap for the next generation of AI.

}, doi = {10.1038/s41467-023-37180-x}, url = {https://www.nature.com/articles/s41467-023-37180-xhttps://www.nature.com/articles/s41467-023-37180-x.pdfhttps://www.nature.com/articles/s41467-023-37180-x.pdfhttps://www.nature.com/articles/s41467-023-37180-x}, author = {Zador, Anthony and Escola, Sean and Richards, Blake and {\"O}lveczky, Bence and Bengio, Yoshua and Boahen, Kwabena and Botvinick, Matthew and Chklovskii, Dmitri and Churchland, Anne and Clopath, Claudia and DiCarlo, James and Ganguli, Surya and Hawkins, Jeff and {\"o}rding, Konrad and Koulakov, Alexei and LeCun, Yann and Lillicrap, Timothy and Marblestone, Adam and Olshausen, Bruno and Pouget, Alexandre and Savin, Cristina and Sejnowski, Terrence and Simoncelli, Eero and Solla, Sara and Sussillo, David and Tolias, Andreas S. and Tsao, Doris} } @article {170, title = {Adversarially trained neural representations may already be as robust as corresponding biological neural representations}, journal = {arXiv}, year = {2022}, month = {06/19/2022}, type = {preprint}, abstract = {

Visual systems of primates are the gold standard of robust perception. There is thus a general belief that mimicking the neural representations that underlie those systems will yield artificial visual systems that are adversarially robust. In this work, we develop a method for performing adversarial visual attacks directly on primate brain activity. We then leverage this method to demonstrate that the above-mentioned belief might not be well founded. Specifically, we report that the biological neurons that make up visual systems of primates exhibit susceptibility to adversarial perturbations that is comparable in magnitude to existing (robustly trained) artificial neural networks.

}, doi = {https://doi.org/10.48550/arXiv.2206.11228}, url = {https://arxiv.org/abs/2206.11228v1}, author = {Guo, Chong and Lee, Michael J and Leclerc, Guillaume and Dapello, Joel and Rao, Yug and Madry, Aleksander and DiCarlo, James J} } @article {162, title = {Chronically implantable LED arrays for behavioral optogenetics in primates}, journal = {Nature Methods}, volume = {18}, year = {2021}, month = {Jan-09-2021}, pages = {1112 - 1116}, abstract = {

Optogenetic methods have been widely used in rodent brains, but remain relatively under-developed for nonhuman primates such as rhesus macaques, an animal model with a large brain expressing sophisticated sensory, motor and cognitive behaviors. To address challenges in behavioral optogenetics in large brains, we developed Opto-Array, a chronically implantable array of light-emitting diodes for high-throughput optogenetic perturbation. We demonstrated that optogenetic silencing in the macaque primary visual cortex with the help of the Opto-Array results in reliable retinotopic visual deficits in a luminance discrimination task. We separately confirmed that Opto-Array illumination results in local neural silencing, and that behavioral effects are not due to tissue heating. These results demonstrate the effectiveness of the Opto-Array for behavioral optogenetic applications in large brains.

}, issn = {1548-7091}, doi = {10.1038/s41592-021-01238-9}, url = {https://www.nature.com/articles/s41592-021-01238-9}, author = {Rajalingham, Rishi and Sorenson, Michael and Azadi, Reza and Bohn, Simon and DiCarlo, James J. and Afraz, Arash} } @article {149, title = {Chronically implantable LED arrays for behavioral optogenetics in primates}, journal = {bioRxiv}, year = {2020}, month = {9/11/2020}, type = {preprint}, abstract = {

Challenges in behavioral optogenetics in large brains demand development of a chronically implantable platform for light delivery. We have developed Opto-Array, a chronically implantable array of LEDs for high-throughput optogenetic perturbation in non-human primates. We tested the Opto-Array in the primary visual cortex of a macaque monkey, and demonstrated that optogenetic cortical silencing by the Opto-Array results in reliable retinotopic visual deficits on a luminance discrimination task.

}, doi = {10.1101/2020.09.10.291583}, url = {https://www.biorxiv.org/content/10.1101/2020.09.10.291583v1.abstract}, author = {Rajalingham, Rish and Sorenson, Michael and Azadi, Reza and Bohn, Simon and DiCarlo, James J. and Afraz, Arash} } @article {141, title = {The inferior temporal cortex is a potential cortical precursor of orthographic processing in untrained monkeys}, journal = {Nature Communications}, volume = {11}, year = {2020}, month = {Jan-12-2020}, abstract = {

The ability to recognize written letter strings is foundational to human reading, but the underlying neuronal mechanisms remain largely unknown. Recent behavioral research in baboons suggests that non-human primates may provide an opportunity to investigate this question. We recorded the activity of hundreds of neurons in V4 and the inferior temporal cortex (IT) while na\ïve macaque monkeys passively viewed images of letters, English words and non-word strings, and tested the capacity of those neuronal representations to support a battery of orthographic processing tasks. We found that simple linear read-outs of IT (but not V4) population responses achieved high performance on all tested tasks, even matching the performance and error patterns of baboons on word classification. These results show that the IT cortex of untrained primates can serve as a precursor of orthographic processing, suggesting that the acquisition of reading in humans relies on the recycling of a brain network evolved for other visual functions.

}, doi = {10.1038/s41467-020-17714-3}, url = {http://www.nature.com/articles/s41467-020-17714-3}, author = {Rajalingham, Rishi and Kar, Kohitij and Sanghavi, Sachi and Dehaene, Stanislas and DiCarlo, James J.} } @article {152, title = {An Open Resource for Non-human Primate Optogenetics}, journal = {Neuron}, year = {2020}, month = {January 10, 2020}, type = {NeuroResource}, abstract = {

Optogenetics has revolutionized neuroscience in small laboratory animals, but its effect on animal models more closely related to humans, such as non-human primates (NHPs), has been mixed. To make evidence-based decisions in primate optogenetics, the scientific community would benefit from a centralized database listing all attempts, successful and unsuccessful, of using optogenetics in the primate brain. We contacted members of the community to ask for their contributions to an open science initiative. As of this writing, 45 laboratories around the world contributed more than 1,000 injection experiments, including precise details regarding their methods and outcomes. Of those entries, more than half had not been published. The resource is free for everyone to consult and contribute to on the Open Science Framework website. Here we review some of the insights from this initial release of the database and discuss methodological considerations to improve the success of optogenetic experiments in NHPs.

}, issn = {08966273}, doi = {10.1016/j.neuron.2020.09.027}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0896627320307510}, author = {Tremblay, Sebastien and Acker, Leah and Afraz, Arash and Albaugh, Daniel L. and Amita, Hidetoshi and Andrei, Ariana R. and Angelucci, Alessandra and Aschner, Amir and Balan, Puiu F. and Basso, Michele A. and Benvenuti, Giacomo and Bohlen, Martin O. and Caiola, Michael J. and Calcedo, Roberto and Cavanaugh, James and Chen, Yuzhi and Chen, Spencer and Chernov, Mykyta M. and Clark, Andrew M. and Dai, Ji and Debes, Samantha R. and Deisseroth, Karl and Desimone, Robert and Dragoi, Valentin and Egger, Seth W. and Eldridge, Mark A.G. and El-Nahal, Hala G. and Fabbrini, Francesco and Federer, Frederick and Fetsch, Christopher R. and Fortuna, Michal G. and Friedman, Robert M. and Fujii, Naotaka and Gail, Alexander and Galvan, Adriana and Ghosh, Supriya and Gieselmann, Marc Alwin and Gulli, Roberto A. and Hikosaka, Okihide and Hosseini, Eghbal A. and Hu, Xing and {\"u}er, Janina and Inoue, Ken-ichi and Janz, Roger and Jazayeri, Mehrdad and Jiang, Rundong and Ju, Niansheng and Kar, Kohitij and Klein, Carsten and Kohn, Adam and Komatsu, Misako and Maeda, Kazutaka and Martinez-Trujillo, Julio C. and Matsumoto, Masayuki and Maunsell, John H.R. and Mendoza-Halliday, Diego and Monosov, Ilya E. and Muers, Ross S. and Nurminen, Lauri and Ortiz-Rios, Michael and {\textquoteright}Shea, Daniel J. and Palfi, {\'e}phane and Petkov, Christopher I. and Pojoga, Sorin and Rajalingham, Rishi and Ramakrishnan, Charu and Remington, Evan D. and Revsine, Cambria and Roe, Anna W. and Sabes, Philip N. and Saunders, Richard C. and Scherberger, {\"o}rg and Schmid, Michael C. and Schultz, Wolfram and Seidemann, Eyal and Senova, Yann-Suhan and Shadlen, Michael N. and Sheinberg, David L. and Siu, Caitlin and Smith, Yoland and Solomon, Selina S. and Sommer, Marc A. and Spudich, John L. and Stauffer, William R. and Takada, Masahiko and Tang, Shiming and Thiele, Alexander and Treue, Stefan and Vanduffel, Wim and Vogels, Rufin and Whitmire, Matthew P. and Wichmann, Thomas and Wurtz, Robert H. and Xu, Haoran and Yazdan-Shahmorad, Azadeh and Shenoy, Krishna V. and DiCarlo, James J. and Platt, Michael L.} } @conference {11, title = {Brain-Like Object Recognition with High-Performing Shallow Recurrent ANNs}, booktitle = {Neural Information Processing Systems}, year = {2019}, abstract = {

Deep convolutional artificial neural networks (ANNs) are the leading class of candidate models of the mechanisms of visual processing in the primate ventral stream. While initially inspired by brain anatomy, over the past years, these ANNs have evolved from a simple eight-layer architecture in AlexNet to extremely deep and branching architectures, demonstrating increasingly better object categorization performance, yet bringing into question how brain-like they still are. In particular, typical deep models from the machine learning community are often hard to map onto the brain\&$\#$39;s anatomy due to their vast number of layers and missing biologically-important connections, such as recurrence. Here we demonstrate that better anatomical alignment to the brain and high performance on machine learning as well as neuroscience measures do not have to be in contradiction. We developed CORnet-S, a shallow ANN with four anatomically mapped areas and recurrent connectivity, guided by Brain-Score, a new large-scale composite of neural and behavioral benchmarks for quantifying the functional fidelity of models of the primate ventral visual stream. Despite being significantly shallower than most models, CORnet-S is the top model on Brain-Score and outperforms similarly compact models on ImageNet. Moreover, our extensive analyses of CORnet-S circuitry variants reveal that recurrence is the main predictive factor of both Brain-Score and ImageNet top-1 performance. Finally, we report that the temporal evolution of the CORnet-S \"IT\" neural population resembles the actual monkey IT population dynamics. Taken together, these results establish CORnet-S, a compact, recurrent ANN, as the current best model of the primate ventral visual stream.

}, doi = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns}, url = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns.pdf}, author = {Jonas Kubilius and Martin Schrimpf and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kohitij Kar and Bashivan, Pouya and Jonathan Prescott-Roy and Kailyn Schmidt and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and James J. DiCarlo} } @inbook {12, title = {Funcitional Properties of Circuits, Cellular Populations, and Areas}, booktitle = {The Neocortex}, volume = {27}, year = {2019}, pages = {223-265}, publisher = {The MIT Press}, organization = {The MIT Press}, chapter = {13}, address = {Cambridge, MA}, issn = {978-0-262-04324-3}, doi = {10.7551/mitpress/12593.001.0001}, url = {http://www.esforum.de}, author = {Harris, KD and Groh, JM and DiCarlo, JJ and Fries, P and Kaschube, M and Laurent, G and MacLean, JN and McCormick, DA and Pipa, G and Reynolds, JN and Schwartz, AB and Sejnowksi, TJ and Singer, W and Vinck, M}, editor = {Singer, Wolf and Sejnowski, Terrence J. and Rakic, Pasko} } @article {10, title = {Reversible Inactivation of Different Millimeter-Scale Regions of Primate IT Results in Different Patterns of Core Object Recognition Deficits}, journal = {Neuron}, volume = {102}, year = {2019}, month = {01/2019}, pages = {493 - 505.e5}, abstract = {

Extensive research suggests that the inferior temporal (IT) population supports visual object recognition\ behavior. However, causal evidence for this hypothesis has been equivocal, particularly beyond the specific case of face-selective subregions of IT. Here, we\ directly tested this hypothesis by pharmacologically inactivating individual, millimeter-scale subregions of IT while monkeys performed several core object recognition subtasks, interleaved trial-by trial. First,\ we observed that IT inactivation resulted in reliable contralateral-biased subtask-selective\ behavioral deficits. Moreover, inactivating different IT subregions resulted in different patterns of subtask deficits, predicted by each subregion\’s neuronal object discriminability. Finally, the similarity between different inactivation effects was tightly related to the\ anatomical distance between corresponding inactivation sites. Taken together, these results provide direct evidence that the IT cortex causally supports general core object recognition and that the underlying IT coding dimensions are topographically organized.

}, issn = {08966273}, doi = {10.1016/j.neuron.2019.02.001}, url = {https://www.cell.com/neuron/pdfExtended/S0896-6273(19)30110-2}, author = {Rajalingham, Rishi and DiCarlo, James J.} } @conference {166, title = {Using Brain-Score to Evaluate and Build Neural Networks for Brain-Like Object Recognition}, booktitle = {Computational and Systems Neuroscience (COSYNE)}, year = {2019}, address = {Denver, CO}, author = {Schrimpf, Martin and Kubilius, Jonas and Hong, Ha and Majaj, Najib and Rajalingham, Rishi and Issa, Elias B and Kar, Kohitij and Ziemba, Corey M and Bashivan, Pouya and Prescott-Roy, Jonathan and Schmidt, Kailyn and Yamins, Daniel LK and DiCarlo, James J} } @article {22, title = {Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?}, journal = {bioRxiv}, year = {2018}, month = {09/2018}, type = {preprint}, abstract = {

The internal representations of early deep artificial neural networks (ANNs) were found to be remarkably similar to the internal neural representations measured experimentally in the primate brain. Here we ask, as deep ANNs have continued to evolve, are they becoming more or less brain-like? ANNs that are most functionally similar to the brain will contain mechanisms that are most like those used by the brain. We therefore developed Brain-Score - a composite of multiple neural and behavioral benchmarks that score any ANN on how similar it is to the brain\&$\#$39;s mechanisms for core object recognition - and we deployed it to evaluate a wide range of state-of-the-art deep ANNs. Using this scoring system, we here report that: (1) DenseNet-169, CORnet-S and ResNet-101 are the most brain-like ANNs. (2) There remains considerable variability in neural and behavioral responses that is not predicted by any ANN, suggesting that no ANN model has yet captured all the relevant mechanisms. (3) Extending prior work, we found that gains in ANN ImageNet performance led to gains on Brain-Score. However, correlation weakened at \>= 70\% top-1 ImageNet performance, suggesting that additional guidance from neuroscience is needed to make further advances in capturing brain mechanisms. (4) We uncovered smaller (i.e. less complex) ANNs that are more brain-like than many of the best-performing ImageNet models, which suggests the opportunity to simplify ANNs to better understand the ventral stream. The scoring system used here is far from complete. However, we propose that evaluating and tracking model-benchmark correspondences through a Brain-Score that is regularly updated with new brain data is an exciting opportunity: experimental benchmarks can be used to guide machine network evolution, and machine networks are mechanistic hypotheses of the brain\&$\#$39;s network and thus drive next experiments. To facilitate both of these, we release Brain-Score.org: a platform that hosts the neural and behavioral benchmarks, where ANNs for visual processing can be submitted to receive a Brain-Score and their rank relative to other models, and where new experimental data can be naturally incorporated.

}, doi = {https://doi.org/10.1101/407007}, url = {https://www.biorxiv.org/content/10.1101/407007v2.full.pdf}, author = {Martin Schrimpf and Kubilius, Jonas and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kar, Kohitij and Bashivan, Pouya and Jonathan Prescott-Roy and Schmidt, Kailyn and Daniel L. K. Yamins and DiCarlo, James J.} } @article {13, title = {Large-Scale, High-Resolution Comparison of the Core Visual Object Recognition Behavior of Humans, Monkeys, and State-of-the-Art Deep Artificial Neural Networks}, journal = {The Journal of Neuroscience}, volume = {38}, year = {2018}, month = {03/2019}, pages = {7255 - 7269}, abstract = {

Primates-including humans-can typically recognize objects in visual images at a glance even in spite of naturally occurring identity-preserving image transformations (e.g. changes in viewpoint). A primary neuroscience goal is to uncover neuron-level mechanistic models that quantitatively explain this behavior by predicting primate performance for each and every image. Here, we applied this stringent behavioral prediction test to the leading mechanistic models of primate vision (specifically, deep, convolutional, artificial neural networks; ANNs) by directly comparing their behavioral signatures against those of humans and rhesus macaque monkeys. Using high-throughput data collection systems for human and monkey psychophysics, we collected over one million behavioral trials from 1472 anonymous humans and five male macaque monkeys for 2400 images over 276 binary object discrimination tasks. Consistent with previous work, we observed that state-of-the-art deep, feed-forward convolutional ANNs trained for visual categorization (termed DCNN models) accurately predicted primate patterns of object-level confusion. However, when we examined behavioral performance for individual images within each object discrimination task, we found that all tested DCNN models were significantly non-predictive of primate performance, and that this prediction failure was not accounted for by simple image attributes, nor rescued by simple model modifications. These results show that current DCNN models cannot account for the image-level behavioral patterns of primates, and that new ANN models are needed to more precisely capture the neural mechanisms underlying primate object vision. To this end, large-scale, high-resolution primate behavioral benchmarks-such as those obtained here-could serve as direct guides for discovering such models.Recently, specific feed-forward deep convolutional artificial neural networks (ANNs) models have dramatically advanced our quantitative understanding of the neural mechanisms underlying primate core object recognition. In this work, we tested the limits of those ANNs by systematically comparing the behavioral responses of these models with the behavioral responses of humans and monkeys, at the resolution of individual images. Using these high-resolution metrics, we found that all tested ANN models significantly diverged from primate behavior. Going forward, these high-resolution, large-scale primate behavioral benchmarks could serve as direct guides for discovering better ANN models of the primate visual system.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0388-18.2018}, url = {http://www.jneurosci.org/content/38/33/7255}, author = {Rajalingham, Rishi and Issa, Elias B. and Bashivan, Pouya and Kar, Kohitij and Schmidt, Kailyn and DiCarlo, James J.} } @article {21, title = {Large-scale, high-resolution comparison of the core visual object recognition behavior of humans, monkeys, and state-of-the-art deep artificial neural networks}, journal = {bioRxiv}, year = {2018}, month = {02/2018}, type = {preprint}, abstract = {

Primates\—including humans\—can typically recognize objects in visual images at a glance even in the face of naturally occurring identity-preserving image transformations (e.g. changes in viewpoint). A primary neuroscience goal is to uncover neuron-level mechanistic models that quantitatively explain this behavior by predicting primate performance for each and every image. Here, we applied this stringent behavioral prediction test to the leading mechanistic models of primate vision (specifically, deep, convolutional, artificial neural networks; ANNs) by directly comparing their behavioral signatures against those of humans and rhesus macaque monkeys. Using high-throughput data collection systems for human and monkey psychophysics, we collected over one million behavioral trials for 2400 images over 276 binary object discrimination tasks. Consistent with previous work, we observed that state-of-the-art deep, feed-forward convolutional ANNs trained for visual categorization (termed DCNNIC\ models) accurately predicted primate patterns of object-level confusion. However, when we examined behavioral performance for individual images within each object discrimination task, we found that all tested DCNNIC\ models were significantly non-predictive of primate performance, and that this prediction failure was not accounted for by simple image attributes, nor rescued by simple model modifications. These results show that current DCNNIC\ models cannot account for the image-level behavioral patterns of primates, and that new ANN models are needed to more precisely capture the neural mechanisms underlying primate object vision. To this end, large-scale, high-resolution primate behavioral benchmarks\—such as those obtained here\—could serve as direct guides for discovering such models.

}, doi = {https://doi.org/10.1101/240614}, url = {https://www.biorxiv.org/content/10.1101/240614v4.full.pdf}, author = {Rajalingham, Rishi and Issa, Elias B and Bashivan, Pouya and Kar, Kohitij and Schmidt, Kailyn and DiCarlo, James J.} } @article {17, title = {Reversible inactivation of different millimeter-scale regions of primate IT results in different patterns of core object recognition deficits}, journal = {bioRxiv}, year = {2018}, month = {08/2018}, type = {preprint}, abstract = {

Extensive research suggests that the inferior temporal (IT) population supports visual object recognition behavior. However, causal evidence for this hypothesis has been equivocal, particularly beyond the specific case of face-selective sub-regions of IT. Here, we directly tested this hypothesis by pharmacologically inactivating individual, millimeter-scale sub-regions of IT while monkeys performed several object discrimination tasks, interleaved trial-by-trial. First, we observed that IT inactivation resulted in reliable contralateral-biased task-selective behavioral deficits. Moreover, inactivating different IT sub-regions resulted in different patterns of task deficits, each predicted by that sub-region\&$\#$39;s neuronal object discriminability. Finally, the similarity between different inactivation effects was tightly related to the anatomical distance between corresponding inactivation sites. Taken together, these results provide direct evidence that IT cortex causally supports general core object recognition, and that the underlying IT codes are topographically organized.

}, doi = {https://doi.org/10.1101/390245}, url = {https://www.biorxiv.org/content/10.1101/390245v1.full.pdf}, author = {Rajalingham, Rishi and DiCarlo, James J.} } @article {29, title = {Comparison of Object Recognition Behavior in Human and Monkey}, journal = {Journal of Neuroscience}, volume = {35}, year = {2015}, month = {02/2015}, pages = {12127 - 12136}, abstract = {

Although the rhesus monkey is used widely as an animal model of human visual processing, it is not known whether invariant visual object recognition behavior is quantitatively comparable across monkeys and humans. To address this question, we systematically compared the core object recognition behavior of two monkeys with that of human subjects. To test true object recognition behavior (rather than image matching), we generated several thousand naturalistic synthetic images of 24 basic-level objects with high variation in viewing parameters and image background. Monkeys were trained to perform binary object recognition tasks on a match-to-sample paradigm. Data from 605 human subjects performing the same tasks on Mechanical Turk were aggregated to characterize \"pooled human\" object recognition behavior, as well as 33 separate Mechanical Turk subjects to characterize individual human subject behavior. Our results show that monkeys learn each new object in a few days, after which they not only match mean human performance but show a pattern of object confusion that is highly correlated with pooled human confusion patterns and is statistically indistinguishable from individual human subjects. Importantly, this shared human and monkey pattern of 3D object confusion is not shared with low-level visual representations (pixels, V1+; models of the retina and primary visual cortex) but is shared with a state-of-the-art computer vision feature representation. Together, these results are consistent with the hypothesis that rhesus monkeys and humans share a common neural shape representation that directly supports object perception.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0573-15.2015}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.0573-15.2015}, author = {Rajalingham, R. and Schmidt, K. and DiCarlo, J. J.} } @article {40, title = {Balanced Increases in Selectivity and Tolerance Produce Constant Sparseness along the Ventral Visual Stream}, journal = {Journal of Neuroscience}, volume = {32}, year = {2012}, month = {07/2012}, pages = {10170 - 10182}, abstract = {

Although popular accounts suggest that neurons along the ventral visual processing stream become increasingly selective for particular objects, this appears at odds with the fact that inferior temporal cortical (IT) neurons are broadly tuned. To explore this apparent contradiction, we compared processing in two ventral stream stages (visual cortical areas V4 and IT) in the rhesus macaque monkey. We confirmed that IT neurons are indeed more selective for conjunctions of visual features than V4 neurons and that this increase in feature conjunction selectivity is accompanied by an increase in tolerance (\"invariance\") to identity-preserving transformations (e.g., shifting, scaling) of those features. We report here that V4 and IT neurons are, on average, tightly matched in their tuning breadth for natural images (\"sparseness\") and that the average V4 or IT neuron will produce a robust firing rate response (\>50\% of its peak observed firing rate) to \∼10\% of all natural images. We also observed that sparseness was positively correlated with conjunction selectivity and negatively correlated with tolerance within both V4 and IT, consistent with selectivity-building and invariance-building computations that offset one another to produce sparseness. Our results imply that the conjunction-selectivity-building and invariance-building computations necessary to support object recognition are implemented in a balanced manner to maintain sparseness at each stage of processing.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.6125-11.2012}, url = {http://www.jneurosci.org/content/32/30/10170.full.pdf+html}, author = {Rust, N. C. and DiCarlo, J. J.} } @article {41, title = {How Does the Brain Solve Visual Object Recognition?}, journal = {Neuron}, volume = {73}, year = {2012}, month = {01/2012}, pages = {415 - 434}, abstract = {

Mounting evidence suggests that \&$\#$39;core object recognition,\&$\#$39; the ability to rapidly recognize objects despite substantial appearance variation, is solved in the brain via a cascade of reflexive, largely feedforward computations that culminate in a powerful neuronal representation in the inferior temporal cortex. However, the algorithm that produces this solution remains poorly understood. Here we review evidence ranging from\ individual neurons and neuronal populations to behavior and computational models. We propose that understanding this algorithm will require using neuronal and psychophysical data to sift through many computational models, each based on building blocks of small, canonical subnetworks with a common functional goal.

}, issn = {08966273}, doi = {10.1016/j.neuron.2012.01.010}, url = {https://www.sciencedirect.com/science/article/pii/S089662731200092X}, author = {DiCarlo, ~J. and Zoccolan, Davide and Rust, ~C.} } @article {45, title = {Selectivity and Tolerance ("Invariance") Both Increase as Visual Information Propagates from Cortical Area V4 to IT}, journal = {Journal of Neuroscience}, volume = {30}, year = {2010}, month = {09/2010}, pages = {12978 - 12995}, abstract = {

Our ability to recognize objects despite large changes in position, size, and context is achieved through computations that are thought to increase both the shape selectivity and the tolerance (\"invariance\") of the visual representation at successive stages of the ventral pathway [visual cortical areas V1, V2, and V4 and inferior temporal cortex {(IT)].} However, these ideas have proven difficult to test. Here, we consider how well population activity patterns at two stages of the ventral stream {(V4} and {IT)} discriminate between, and generalize across, different images. We found that both V4 and {IT} encode natural images with similar fidelity, whereas the {IT} population is much more sensitive to controlled, statistical scrambling of those images. Scrambling sensitivity was proportional to receptive field {(RF)} size in both V4 and {IT,} suggesting that, on average, the number of visual feature conjunctions implemented by a V4 or {IT} neuron is directly related to its {RF} size. We also found that the {IT} population could better discriminate between objects across changes in position, scale, and context, thus directly demonstrating a {V4-to-IT} gain in tolerance. This tolerance gain could be accounted for by both a decrease in single-unit sensitivity to identity-preserving transformations (e.g., an increase in {RF} size) and an increase in the maintenance of rank-order object selectivity within the {RF.} These results demonstrate that, as visual information travels from V4 to {IT,} the population representation is reformatted to become more selective for feature conjunctions and more tolerant to identity preserving transformations, and they reveal the single-unit response properties that underlie that reformatting.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0179-10.2010}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.0179-10.2010}, author = {Rust, N. C. and DiCarlo, J. J.} } @article {55, title = {Balanced increases in selectivity and invariance produce constant sparseness across the ventral visual pathway}, journal = {Journal of Vision}, volume = {9}, year = {2009}, month = {Jan-08-2009}, pages = {738 - 738}, abstract = {

While several studies report neurons in inferotemporal cortex (IT) that are highly selective for particular objects or images, other studies report that neurons in IT tend to be broadly tuned. To investigate how selectivity changes across the ventral visual pathway, we compared the responses of neurons in a mid-level visual area (V4) and a high-level visual area (IT). We first assessed the selectivity of neurons in each area by determining how well each population could discriminate between natural images and \“scrambled\” versions of those images that have the same low-level structure but configured randomly. We found that the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled images. These results suggest that IT neurons are in fact more selective than V4 neurons in terms of the image features that drive these cells. As a second estimate of selectivity, we measured the tuning bandwidth of neurons for natural images (\“sparseness\”). Surprisingly, we found that distributions of sparseness values were indistinguishable between V4 and IT. How can the selectivity for natural image features increase while the tuning bandwidth for natural images remains constant? One possibility is that increases in selectivity for particular image features are offset by increases in tolerance for the (e.g.) position and scale of those features. We found that indeed, measures of tolerance were higher in IT than V4. These results confirm that neurons increase both their selectivity for image features and their tolerance to changes in the position and scale of those features as signals propagate through the ventral pathway. Remarkably, the rates of increase of these two parameters appear to be set such that the tuning bandwidth for natural images is maintained across each stage of cortical processing.

}, doi = {10.1167/9.8.738}, url = {http://jov.arvojournals.org/Article.aspx?doi=10.1167/9.8.738}, author = {Rust, N. C. and DiCarlo, J. J.} } @proceedings {69, title = {Concurrent increases in selectivity and tolerance produce constant sparseness across the ventral visual stream}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2008}, month = {03/2008}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, abstract = {

Neural coding schemes that minimize the number of neurons activated at any one time (or equivalently maximize \“sparseness\”) are thought to be both metabolically and computationally efficient [reviewed by 1]. But does sparseness increase as signals propagate through the cortex? To investigate this question, we compared the response properties of neurons at different stages along the pathway supporting object recognition, the ventral visual stream. Specifically, we recorded the responses of neurons in a mid-level visual area (V4) and a high-level visual area (anterior inferotemporal cortex, IT) to a large set of natural images while monkeys performed an object detection task. We found that the distributions of sparseness values in V4 and IT were indistinguishable and that most neurons in both areas were broadly tuned. Similarly, individual images activated the same, large fraction of neurons in V4 and IT. Thus it appears that a coding principle is conserved at each level of processing; however, in opposition to theories of sparse coding, the tuning we observed in mid- and high-level vision is more consistent with a broadly distributed coding scheme [see also 2].

If sparseness is not changing, what is happening as signals propagate through the visual system? We began investigating this question by measuring tolerance to position and scale transformations. We found that individual neuron tolerance increased from V4 to IT, and this translated to enhanced performance of the IT over the V4 population on a position- and scale-invariant object recognition task. To determine whether the neurons in each area also differ in terms of the image features that elicit a response, we presented natural and \“scrambled\” images that have the same local structure but configured randomly [3]. We found that V4 neurons responded similarly to both image sets whereas IT neurons responded much more robustly to the natural images. Likewise, the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled as compared to the natural images. These results suggest that IT neurons are more selective than V4 neurons in terms of the image features that drive these cells. Moreover, we found that equivalent sparseness values were correlated with higher levels of selectivity and tolerance in IT as compared to V4. Thus, as signals propagate through the visual system, neurons increase their selectivity for particular image features and, at the same time, neurons increase their tolerance for the position and scale of those features; the rates at which these two factors increase are set such that constant sparseness is maintained at each level of visual processing. Consistent with the observation that the structure of cortex is roughly identical regardless of where it sits in the hierarchy, we speculate that conservation of a broadly distributed coding scheme is an optimal use of resources in equipotential cortex.

References

[1] Sparse coding of sensory inputs. BA Olshausen and DJ Field, Curr Opin Neurobiol., 14:481-7, 2004. [2]Responses of neurons in primary and inferior temporal visual cortices to natural scenes. R Baddeley et al., Proc Biol Sci. 264:1775-83, 1997. [3] A parametric texture model based on joint statistics of complex wavelet coefficients. J Portilla and EP Simoncelli, Int J Comp Vis, 40:49-71, 2000

}, url = {http://www.cosyne.org/c/images/8/8e/Cosyne_pf_new.pdf}, author = {Rust, N. C. and DiCarlo, James J.} } @proceedings {67, title = {Increases in selectivity are offset by increases in tolerance ("invariance") to maintain sparseness across the ventral visual pathway}, journal = {Society for Neuroscience}, year = {2008}, month = {11/2008}, pages = {514.8}, publisher = {SFN}, address = {Washington, DC, USA}, abstract = {

Popular accounts of visual processing suggest that neurons become increasingly selective for particular objects and scenes as signals propagate through the ventral visual pathway, but this hypothesis has proven difficult to test. To investigate this issue systematically, we recorded the responses of neurons in a mid-level visual area (V4) and a high-level visual area (IT) under well-controlled conditions (same monkeys, same task, same region of the visual field, same stimuli). We assessed the selectivity of neurons in each area by determining how well each population could discriminate between natural images and \“scrambled\” versions of those images that have the same local structure but configured randomly. We found that the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled as compared to the natural images. These results suggest that IT neurons are more selective than V4 neurons in terms of the image features that drive these cells. As a second estimate of selectivity, we measured the tuning bandwidth of neurons for natural images (commonly called \“sparseness\”). Surprisingly, we found that distributions of sparseness values were indistinguishable between V4 and IT. Similarly, we found that individual images activated the same fraction of neurons in V4 and IT, suggesting that a coding principle is conserved at each level of processing. How can the selectivity for natural image features increase while the tuning bandwidth for natural images remains constant? One possible explanation is that increases in selectivity for particular image features are offset by increases in tolerance for the (e.g.) position and scale of those features. Indeed, when we measured the tolerance of neurons to changes in the position and scale of images, we found that tolerance increases as signals propagate from V4 to IT. Moreover, we found that equivalent sparseness values were correlated with higher levels of selectivity and tolerance in IT as compared to V4. These results confirm that neurons increase both their selectivity for image features and their tolerance to changes in the position and scale of those features as signals propagate through the ventral visual pathway. Remarkably, the rates of increase of these two parameters appear to be set such that an equally distributed coding scheme is maintained at each level of visual processing.

}, url = {https://www.abstractsonline.com/Plan/ViewAbstract.aspx?sKey=fc0d0a2d-b563-4b41-8311-0805f08bde8a\&cKey=8a2c998e-bc76-4d92-96ac-ad5199da59bf\&mKey=\%7bAFEA068D-D012-4520-8E42-10E4D1AF7944\%7d}, author = {Rust, N. C. and James J. DiCarlo} } @proceedings {72, title = {Inferior temporal cortex robustly signals encounters with new objects, but is not an online representation of the visual world}, journal = {Society for Neuroscience}, year = {2008}, month = {11/2008}, pages = {316.6}, publisher = {SFN}, address = {Washington, DC, USA}, abstract = {

Popular accounts of visual processing suggest that neurons become increasingly selective for particular objects and scenes as signals propagate through the ventral visual pathway, but this hypothesis has proven difficult to test. To investigate this issue systematically, we recorded the responses of neurons in a mid-level visual area (V4) and a high-level visual area (IT) under well-controlled conditions (same monkeys, same task, same region of the visual field, same stimuli). We assessed the selectivity of neurons in each area by determining how well each population could discriminate between natural images and \“scrambled\” versions of those images that have the same local structure but configured randomly. We found that the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled as compared to the natural images. These results suggest that IT neurons are more selective than V4 neurons in terms of the image features that drive these cells. As a second estimate of selectivity, we measured the tuning bandwidth of neurons for natural images (commonly called \“sparseness\”). Surprisingly, we found that distributions of sparseness values were indistinguishable between V4 and IT. Similarly, we found that individual images activated the same fraction of neurons in V4 and IT, suggesting that a coding principle is conserved at each level of processing. How can the selectivity for natural image features increase while the tuning bandwidth for natural images remains constant? One possible explanation is that increases in selectivity for particular image features are offset by increases in tolerance for the (e.g.) position and scale of those features. Indeed, when we measured the tolerance of neurons to changes in the position and scale of images, we found that tolerance increases as signals propagate from V4 to IT. Moreover, we found that equivalent sparseness values were correlated with higher levels of selectivity and tolerance in IT as compared to V4. These results confirm that neurons increase both their selectivity for image features and their tolerance to changes in the position and scale of those features as signals propagate through the ventral visual pathway. Remarkably, the rates of increase of these two parameters appear to be set such that an equally distributed coding scheme is maintained at each level of visual processing.

}, url = {https://www.abstractsonline.com/Plan/ViewAbstract.aspx?sKey=ee83e7f7-5aea-4ec8-a948-436658d20e37\&cKey=fc64f0af-c81e-4b0e-b809-796349279531\&mKey=\%7bAFEA068D-D012-4520-8E42-10E4D1AF7944\%7d}, author = {Rust, N. C. and DiCarlo, James J.} } @proceedings {73, title = {Is the rodent a valuable model system for studying invariant object recognition?}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2008}, month = {03/2008}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, abstract = {

Despite the many advantages rodents offer in terms of experimental accessibility, they have never been extensively used as models to investigate the neuronal processing of visual objects. A crucial step in establishing whether rodents are suitable models for the study of object vision, is to assess if they are capable of invariant object recognition \– i.e., recognition of visual objects across the range of transformations that they typically undergo during natural vision (e.g., changes in position and size). In this study, we tested rats (Long-Evans) capability to discriminate between two geometrical shapes (square and triangle) presented at different sizes, positions and orientations. In each trial, a single shape was presented on a computer monitor and the animal had to report its identity. The experiment consisted of three phases. In phase I, each animal learned to discriminate between the two shapes presented at fixed size (40\° of visual angle), position (center of the monitor), and orientation. Na\ïve rats typically achieved \> 70\% correct performance in 2-3 weeks of training. In phase II, the animals were trained to perform the task while either the size, the horizontal position, or the orientation of the shapes was separately varied. Rats readily acquired this task (\> 70\% correct performance) across sizes ranging from 40\° to 10\°, positions spanning \±12\°, and orientations spanning \±40\°. During phase III of the experiment, we asked if rats could generalize to novel combinations of size, position, and orientation (a total of 100 transformations of each target shape were tested). Performance was typically \> 70\% correct for nearly all of these previously unseen transformations (performance of one rat over a subset of the tested conditions is shown in the figure) and even for a fraction of transformations for which feedback was withheld. These results show that rats can readily learn to: 1) discriminate between simple visual shapes; 2) disregard variations in their position, size and orientation; and 3) generalize to novel views. This suggests that the rat visual system contains the fundamental mechanisms that support object recognition. Therefore, given the broad ranges of experimental approaches available in rats, they may be a powerful new model system to study the neuronal basis of invariant object recognition

}, url = {http://www.cosyne.org/c/images/8/8e/Cosyne_pf_new.pdf}, author = {Zoccolan, Davide and Cox, David D. and Oertelt, Nadja and Radwam, Basma and Tsang, Sabrina and DiCarlo, James J.} }