@article {177, title = {A Unifying Principle for the Functional Organization of Visual Cortex}, journal = {bioRxiv}, year = {2023}, month = {2023}, abstract = {

A key feature of many cortical systems is functional organization: the arrangement of neurons with specific functional properties in characteristic spatial patterns across the cortical surface. However, the principles\ underlying the emergence and utility of functional organization are poorly understood. Here we develop\ the Topographic Deep Artificial Neural Network (TDANN), the first unified model to accurately predict the\ functional organization of multiple cortical areas in the primate visual system. We analyze the key factorsresponsible for the TDANN\’s success and find that it strikes a balance between two specific objectives:achieving a task-general sensory representation that is self-supervised, and maximizing the smoothness of\ responses across the cortical sheet according to a metric that scales relative to cortical surface area. In\ turn, the representations learned by the TDANN are lower dimensional and more brain-like than those in\ models that lack a spatial smoothness constraint. Finally, we provide evidence that the TDANN\’s functionalorganization balances performance with inter-area connection length, and use the resulting models for\ a proof-of-principle optimization of cortical prosthetic design. Our results thus offer a unified principle\ for understanding functional organization and a novel view of the functional role of the visual system in\ particular.

}, doi = { https://doi.org/10.1101/2023.05.18.541361}, url = {https://www.biorxiv.org/content/10.1101/2023.05.18.541361v1.full.pdf}, author = {Margalit, Eshed and Lee, Hyodong and Finzi, Dawn and DiCarlo, James J and Grill-Spector, Kalanit and Yamins, Daniel LK} } @article {160, title = {Unsupervised changes in core object recognition behavior are predicted by neural plasticity in inferior temporal cortex}, journal = {eLife}, volume = {10}, year = {2021}, month = {Nov-06-2021}, abstract = {

Temporal continuity of object identity is a feature of natural visual input, and is potentially exploited -- in an unsupervised manner -- by the ventral visual stream to build the neural representation in inferior temporal (IT) cortex. Here we investigated whether plasticity of individual IT neurons underlies human core-object-recognition behavioral changes induced with unsupervised visual experience. We built a single-neuron plasticity model combined with a previously established IT population-to-recognition-behavior linking model to predict human learning effects. We found that our model, after constrained by neurophysiological data, largely predicted the mean direction, magnitude and time course of human performance changes. We also found a previously unreported dependency of the observed human performance change on the initial task difficulty. This result adds support to the hypothesis that tolerant core object recognition in human and non-human primates is instructed -- at least in part -- by naturally occurring unsupervised temporal contiguity experience.

}, doi = {10.7554/eLife.60830}, url = {https://elifesciences.org/articles/60830https://cdn.elifesciences.org/articles/60830/elife-60830-v2.pdfhttps://cdn.elifesciences.org/articles/60830/elife-60830-v2.xml}, author = {Jia, Xiaoxuan and Hong, Ha and DiCarlo, James J} } @article {167, title = {Unsupervised neural network models of the ventral visual stream}, journal = {Proceedings of the National Academy of Sciences}, volume = {118}, year = {2021}, month = {Jul-01-2022}, pages = {e2014196118}, abstract = {

Deep neural networks currently provide the best quantitative models of the response patterns of neurons throughout the primate ventral visual stream. However, such networks have remained implausible as a model of the development of the ventral stream, in part because they are trained with supervised methods requiring many more labels than are accessible to infants during development. Here, we report that recent rapid progress in unsupervised learning has largely closed this gap. We find that neural network models learned with deep unsupervised contrastive embedding methods achieve neural prediction accuracy in multiple ventral visual cortical areas that equals or exceeds that of models derived using today\’s best supervised methods and that the mapping of these neural network models\’ hidden layers is neuroanatomically consistent across the ventral stream. Strikingly, we find that these methods produce brain-like representations even when trained solely with real human child developmental data collected from head-mounted cameras, despite the fact that these datasets are noisy and limited. We also find that semisupervised deep contrastive embeddings can leverage small numbers of labeled examples to produce representations with substantially improved error-pattern consistency to human behavior. Taken together, these results illustrate a use of unsupervised learning to provide a quantitative model of a multiarea cortical brain system and present a strong candidate for a biologically plausible computational theory of primate sensory learning.

}, issn = {0027-8424}, doi = {10.1073/pnas.2014196118}, url = {http://www.pnas.org/lookup/doi/10.1073/pnas.2014196118}, author = {Zhuang, Chengxu and Yan, Siming and Nayebi, Aran and Schrimpf, Martin and Frank, Michael C. and DiCarlo, James J. and Yamins, Daniel L. K.} } @article {7, title = {Unsupervised changes in core object recognition behavioral performance are accurately predicted by unsupervised neural plasticity in inferior temporal cortex}, journal = {BioRxiv}, year = {2020}, month = {01.2020}, type = {preprint}, abstract = {

Temporal continuity of object identity is a feature of natural visual input, and is potentially exploited -- in an unsupervised manner -- by the ventral visual stream to build the neural representation in inferior temporal (IT) cortex and IT-dependent core object recognition behavior. Here we investigated whether plasticity of individual IT neurons underlies human behavioral changes induced with unsupervised visual experience by building a single-neuron plasticity model combined with a previously established IT population-to-recognition-behavior linking model to predict human learning effects. We found that our model quite accurately predicted the mean direction, magnitude and time course of human performance changes. We also found a previously unreported dependency of the observed human performance change on the initial task difficulty. This result adds support to the hypothesis that tolerant core object recognition in human and non-human primates is instructed -- at least in part -- by naturally occurring unsupervised temporal contiguity experience.

}, doi = {https://doi.org/10.1101/2020.01.13.900837}, url = {https://www.biorxiv.org/content/10.1101/2020.01.13.900837v2.full.pdf}, author = {Xiaoxuan Jia and Ha Hong and James J. DiCarlo} } @article {150, title = {Unsupervised Neural Network Models of the Ventral Visual Stream}, journal = {bioRxiv}, year = {2020}, month = {June 18, 2020}, type = {preprint}, abstract = {

Deep neural networks currently provide the best quantitative models of the response patterns of neurons throughout the primate ventral visual stream. However, such networks have remained implausible as a model of the development of the ventral stream, in part because they are trained with supervised methods requiring many more labels than are accessible to infants during development. Here, we report that recent rapid progress in unsupervised learning has largely closed this gap. We find that neural network models learned with deep unsupervised contrastive embedding methods achieve neural prediction accuracy in multiple ventral visual cortical areas that equals or exceeds that of models derived using today\’s best supervised methods, and that the mapping of these neural network models\’ hidden layers is neuroanatomically consistent across the ventral stream. Moreover, we find that these methods produce brain-like representations even when trained on noisy and limited data measured from real children\’s developmental experience. We also find that semi-supervised deep contrastive embeddings can leverage small numbers of labelled examples to produce representations with substantially improved error-pattern consistency to human behavior. Taken together, these results suggest that deep contrastive embedding objectives may be a biologically-plausible computational theory of primate visual development.

}, doi = {10.1101/2020.06.16.155556}, url = {https://www.biorxiv.org/content/10.1101/2020.06.16.155556v1.abstract}, author = {Zhuang, Chengxu and Yan, Siming and Nayebi, Aran and Schrimpf, Martin and Frank, Michael and DiCarlo, James J. and Yamins, Daniel L.K.} } @conference {166, title = {Using Brain-Score to Evaluate and Build Neural Networks for Brain-Like Object Recognition}, booktitle = {Computational and Systems Neuroscience (COSYNE)}, year = {2019}, address = {Denver, CO}, author = {Schrimpf, Martin and Kubilius, Jonas and Hong, Ha and Majaj, Najib and Rajalingham, Rishi and Issa, Elias B and Kar, Kohitij and Ziemba, Corey M and Bashivan, Pouya and Prescott-Roy, Jonathan and Schmidt, Kailyn and Yamins, Daniel LK and DiCarlo, James J} } @article {26, title = {Using goal-driven deep learning models to understand sensory cortex}, journal = {Nature Neuroscience}, volume = {19}, year = {2016}, month = {01/2016}, pages = {356 - 365}, abstract = {

Fueled by innovation in the computer vision and artificial intelligence communities, recent developments in computational neuroscience have used goal-driven hierarchical convolutional neural networks (HCNNs) to make strides in modeling neural single-unit and population responses in higher visual cortical areas. In this Perspective, we review the recent progress in a broader modeling context and describe some of the key technical innovations that have supported it. We then outline how the goal-driven HCNN approach can be used to delve even more deeply into understanding the development and organization of sensory cortical processing.

}, issn = {1097-6256}, doi = {10.1038/nn.4244}, url = {http://www.nature.com/articles/nn.4244.pdf}, author = {Yamins, Daniel L K and DiCarlo, James J} } @conference {42, title = {A unified neuronal population code fully explains human object recognition.}, booktitle = {Computation and Systems Neuroscience (COSYNE)}, year = {2012}, month = {02/2012}, address = {Salt Lake City, Utah, USA}, abstract = {

Our goal is to understand the neuronal mechanisms that underlie human visual object recognition (OR). While previous work has argued for qualitative links between neuronal responses in the ventral visual stream and human shape judgements, no study has asked which, if any, neuronal responses are quantitatively sufficient to explain broad domain human OR performance. The shift from qualitative to quantitative hypotheses requires a framework to link neuronal responses to behavior (\“unified code\”). Here we ask: is there a common neuronal basis (e.g., in IT cortex) and a simple (e.g., linear) transformation that will predict all of human OR performance? We first defined OR operationally by obtaining human psychophysical measurements using images that explore shape similarity and identity preserving image variation, resulting in OR benchmarks that span a range of difficulty. Using the same visual images, we measured neuronal responses in V4 and IT in two monkeys. We implemented 14 unified codes based on those neuronal data and computed cross-validated neuronal discriminability indices (d\’s) to compare to the human d\’s. The dynamic range across those d\’s sets a high bar for when a putative code is sufficient to explain behavior: it is not sufficient for a code to perform well (high d\’) or to match one d\’. Instead, a sufficient unified code must also emergently predict the entire pattern of behavior over all tasks. Remarkably, we found a few unified IT-based codes that meet this high bar. Interestingly, many other IT codes and all V4 codes are insufficient. While humans outperform computer vision systems on many of our OR tasks, their abilities reliably depend on the images tested. These dependencies in human performance are fully explained by a simple, unified reading of monkey ventral stream neurons, a feat unmatched by any computer vision system we tested

}, doi = {http://www.cosyne.org/c/index.php?title=Cosyne_12}, url = {http://cosyne.org/cosyne12/Cosyne2012_program_book.pdf}, author = {Majaj, Najib J and Hong, Ha and Solomon, EA and DiCarlo, James J.} } @article {46, title = {Unsupervised Natural Visual Experience Rapidly Reshapes Size-Invariant Object Representation in Inferior Temporal Cortex}, journal = {Neuron}, volume = {67}, year = {2010}, month = {01/2010}, pages = {1062 - 1075}, abstract = {

We easily recognize objects and faces across\ a myriad of retinal images produced by each object. One hypothesis is that this tolerance (a.k.a. \"invariance\") is learned by relying on the fact that object identities are temporally stable. While we previously found neuronal evidence supporting this idea at the top of the nonhuman primate ventral visual stream (inferior temporal cortex, or {IT),} we here test if this is a general tolerance learning mechanism. First, we found that the same type of unsupervised experience that reshaped {IT} position tolerance also predictably reshaped {IT} size tolerance, and the magnitude of reshaping was quantitatively similar. Second, this tolerance reshaping can be induced under naturally occurring dynamic visual experience, even without eye movements. Third, unsupervised temporal contiguous experience can build new neuronal tolerance. These results suggest that the ventral visual stream uses a general unsupervised tolerance learning algorithm to build its invariant object representation.

}, issn = {08966273}, doi = {10.1016/j.neuron.2010.08.029}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0896627310006392}, author = {Li, Nuo and DiCarlo, James J.} } @proceedings {60, title = {Unlocking Biologically-Inspired Computer Vision: a High-Throughput Approach}, journal = {NVIDIA GPU Technology Conference}, year = {2009}, month = {2009}, author = {Pinto, Nicolas and Cox, David D. and DiCarlo, James J.} } @proceedings {61, title = {Unlocking Brain-Inspired Computer Vision}, journal = {GPU@BU}, year = {2009}, month = {2009}, author = {Pinto, Nicolas and Cox, David D. and DiCarlo, James J.} } @proceedings {76, title = {Unsupervised natural experience rapidly alters invariant object representation in visual cortex}, journal = {Society for Neuroscience}, year = {2008}, month = {11/2008}, pages = {316.5}, publisher = {SFN}, address = {Washington, DC, USA}, abstract = {

The responses of cortical neurons at the top of the ventral visual stream -- inferior temporal (IT) cortex -- are selective to visual objects, yet tolerant (\“invariant\”) to changes in object position, size, pose, etc. Though IT responses likely underlie object recognition behavior, how that neuronal tolerance is constructed remains a fundamental mystery. One possibility is that natural visual experience is an implicit teacher: because objects are present for relatively long time intervals, while object motion or viewer motion (e.g. eye movements) cause rapid changes in each object\’s retinal image, the visual system could learn tolerance by associating neuronal representations that occur closely in time. If this hypothesis is correct, then we might create \“incorrect\” tolerance by engineering an altered visual world in which we temporally couple the retinal images of two different objects at different retinal positions. The main prediction is that the visual system would incorrectly associate the representations of those objects at those positions. Thus, IT neurons might lose their position tolerant selectivity, and instead begin to prefer one object at one position and another object at the other position. To test this idea, two monkeys visually explored our altered visual world and we used real-time eye tracking to present visual objects at controlled retinal positions during free viewing. As the animal saccaded toward a specific object (P), it was consistently replaced by another object (N), rendering the image of P at a peripheral retinal position (\“swapped\”) temporally contiguous with the image of N on the fovea. We found that exposure to these altered statistics changed IT object selectivity specifically at the swapped position, as predicted. This unsupervised temporal tolerance learning (UTL) was substantial (~5 spk/s selectivity change in 1 hr), gradually increased with exposure, and was highly significant at the population level (p=0.007 \“position x exposure\” interaction, bootstrap). Coupled with the finding that this same experience manipulation changes the position tolerance of human object perception (Cox et al, 2005), we speculate that UTL may reflect the mechanism by which the visual system builds and maintains tolerant object representations. The relatively fast time-scale and unsupervised nature of UTL open the door to advances in systematically characterizing the spatiotemporal image statistics that drive it, understanding if it plays a role in other types of tolerance (e.g. pose, scale), and perhaps connecting a central cognitive ability -- tolerant object recognition -- to cellular and molecular plasticity mechanisms.

}, url = {https://www.abstractsonline.com/Plan/ViewAbstract.aspx?sKey=ee83e7f7-5aea-4ec8-a948-436658d20e37\&cKey=9a873eb3-f8d3-48b5-8df9-9f7b2ef0a3d9\&mKey=\%7bAFEA068D-D012-4520-8E42-10E4D1AF7944\%7d}, author = {Li, Nuo and DiCarlo, James J.} } @article {63, title = {Unsupervised Natural Experience Rapidly Alters Invariant Object Representation in Visual Cortex}, journal = {Science}, volume = {321}, year = {2008}, month = {12/2008}, pages = {1502 - 1507}, abstract = {

Object recognition is challenging because each object produces myriad retinal images. Responses of neurons from the inferior temporal cortex {(IT)} are selective to different objects, yet tolerant (\"invariant\") to changes in object position, scale, and pose. How does the brain construct this neuronal tolerance? We report a form of neuronal learning that suggests the underlying solution. Targeted alteration of the natural temporal contiguity of visual experience caused specific changes in {IT} position tolerance. This unsupervised temporal slowness learning {(UTL)} was substantial, increased with experience, and was significant in single {IT} neurons after just 1 hour. Together with previous theoretical work and human object perception experiments, we speculate that {UTL} may reflect the mechanism by which the visual stream builds and maintains tolerant object representations.

}, issn = {0036-8075}, doi = {10.1126/science.1160028}, url = {http://www.sciencemag.org/cgi/content/full/321/5895/1502?ijkey=wb6T4x69JeSes\&keytype=ref\&siteid=sci}, author = {Li, Nuo and DiCarlo, James J.} } @article {79, title = {Untangling invariant object recognition}, journal = {Trends in Cognitive Sciences}, volume = {11}, year = {2007}, month = {01/2007}, pages = {333 - 341}, abstract = {

Despite tremendous variation in the appearance of visual objects, primates can recognize a multitude of objects, each in a fraction of a second, with no apparent effort. However, the brain mechanisms that enable this fundamental ability are not understood. Drawing on ideas from neurophysiology and computation, we present a graphical perspective on the key computational challenges of object recognition, and argue that the format of neuronal population representation and a property that we term \&$\#$39;object tangling\&$\#$39; are central. We use this perspective to show that the primate ventral visual processing stream achieves a particularly effective solution in which single-neuron invariance is not the goal. Finally, we speculate on the key neuronal mechanisms that could enable this solution, which, if understood, would have far-reaching implications for cognitive neuroscience.

}, issn = {13646613}, doi = {10.1016/j.tics.2007.06.010}, url = {https://linkinghub.elsevier.com/retrieve/pii/S1364661307001593}, author = {DiCarlo, James J. and Cox, David D.} } @article {93, title = {Ultra-fast object recognition from few spikes}, number = {2005-022}, year = {2005}, month = {07/2005}, pages = {1-31}, institution = {MIT}, address = {Cambridge, MA}, abstract = {

Understanding the complex brain computations leading to object recognition requires quantitatively characterizing the information represented in inferior temporal cortex (IT), the highest stage of the primate visual stream. A read-out technique based on a trainable classifier is used to characterize the neural coding of selectivity and invariance at the population level. The activity of very small populations of independently recorded IT neurons (~100 randomly selected cells) over very short time intervals (as small as 12.5 ms) contains surprisingly accurate and robust information about both object \Â\‘identity\Â\’ and \Â\‘category\Â\’, which is furthermore highly invariant to object position and scale. Significantly, selectivity and invariance are present even for novel objects, indicating that these properties arise from the intrinsic circuitry and do not require object-specific learning. Within the limits of the technique, there is no detectable difference in the latency or temporal resolution of the IT information supporting so-called \Â\‘categorization\Â\’ (a.k. basic level) and \Â\‘identification\Â\’ (a.k. subordinate level) tasks. Furthermore, where information, in particular information about stimulus location and scale, can also be read-out from the same small population of IT neurons. These results show how it is possible to decode invariant object information rapidly, accurately and robustly from a small population in IT and provide insights into the nature of the neural code for different kinds of object-related information.

}, keywords = {AI, inferior temporal cortex, neural coding, object recognition}, issn = {2005-022}, url = {https://dspace.mit.edu/handle/1721.1/30556}, author = {Hung, Chou P. and Kreiman, Gabriel and Poggio, Tomaso and DiCarlo, James J.} } @proceedings {94, title = {Using {\textquoteleft}read-out{\textquoteright} of object identity to understand object coding in the macaque anterior inferior temporal cortex}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2005}, month = {03/2005}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, abstract = {

Recent efforts to develop robust computer vision systems capable of approaching the level of human object recognition performance have shown that models based on neurobiology can outperform non-biological models. To continue the development of such systems, we must understand the codes used by neuronal ensembles to represent object identity--a matter of continued debate in the neuroscience community. Understanding such codes would also allow the development of brain-machine interfaces capable of \‘reading-out\’ or \‘writing-in\’ information to AIT. One approach to understanding these neuronal codes is to obtain a large sample of neuronal data using a fixed set of objects, and then test the ability of various neuronal population measures (codes) to convey knowledge of object identity \– that is, determine the ability of each putative neuronal code to \‘read-out\’ object identity information from AIT. To this end, we have now recorded single-unit activity (SUA), multiunit activity (MUA) and local field potentials (LFP), from over 500 sites in the anterior inferior temporal cortex (AIT) of two macaque monkeys while they viewed a fixed set of 77 complex objects. In this paper, we focus on the temporal latency and resolution of AIT neuronal population codes that best convey knowledge of object identity as assessed by object classification. Specifically, the 77 objects were divided before the experiment into eight different classes: toys, foodstuffs, human faces, monkey faces, hands, vehicles, boxes and cats/dogs. To evaluate the performance of each putative neuronal population code, we used support vector machines (SVM) and jackknife cross-validation to avoid over-fitting. Under the assumption of no covariance across spatially separate sites, these data and methods allowed us to examine putative neuronal population codes that differed in their latency (0 to 100 ms from stimulus onset) and their temporal resolution (12.5 to 200 ms wide bins). Results were evaluated for consistency across different sets of recording sites. We found that MUA and SUA spike count codes could perform single trial object classification. Performance improved with the number of sites and reached ~90\% when ~128 arbitrary AIT sites were included. For any given number of sites, increasing the temporal resolution beyond ~50 ms did not lead to improved performance and the optimal latency from stimulus onset was ~100 ms. Because the neuronal data were not collected simultaneously, we cannot rule out spatio-temporal codes with a time scale less than ~50 ms. However, these results support the hypothesis that the time scale of the object identity code in AIT is ~50 ms and that downstream read-out of object identity for behavior or further processing may simply integrate over this time scale.\ 

}, author = {Hung, Chou P. and Kreiman, Gabriel and Quiroga, Rodrigo Quian and Kraskov, Alexander and Poggio, Tomaso and DiCarlo, James J.} } @article {89, title = {Using Neuronal Latency to Determine Sensory{\textendash}Motor Processing Pathways in Reaction Time Tasks}, journal = {Journal of Neurophysiology}, volume = {93}, year = {2004}, month = {11/2004}, pages = {2974 - 2986}, abstract = {

We describe a new technique that uses the timing of neuronal and behavioral responses to explore the contributions of individual neurons to specific behaviors. The approach uses both the mean neuronal latency and the trial-by-trial covariance between neuronal latency and behavioral response. Reliable measurements of these values were obtained from single-unit recordings made from anterior inferotemporal (AIT) cortex and the frontal eye fields (FEF) in monkeys while they performed a choice reaction time task. These neurophysiological data show that the responses of AIT neurons and some FEF neurons have little covariance with behavioral response, consistent with a largely \"sensory\" response. The responses of another group of FEF neurons with longer mean latency covary tightly with behavioral response, consistent with a largely \"motor\" response. A very small fraction of FEF neurons had responses consistent with an intermediate position in the sensory-motor pathway. These results suggest that this technique is a valuable tool for exploring the functional organization of neuronal circuits that underlie specific behaviors.

}, keywords = {Action Potentials, Afferent, Animal, Animals, Behavior, Macaca mulatta, Male, Models, Motor Neurons, Neural Pathways, Neurological, Neurons, Photic Stimulation, Psychomotor Performance, Reaction Time, Task Performance and Analysis, Temporal Lobe, Time Factors, Visual Fields}, issn = {0022-3077}, doi = {10.1152/jn.00508.2004}, url = {https://www.physiology.org/doi/10.1152/jn.00508.2004}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @proceedings {101, title = {Using reaction time tasks to map sensory-motor chains in the monkey}, journal = {Society for Neuroscience}, year = {2002}, month = {10/2002}, publisher = {SFN}, address = {Orlando, FL, USA}, author = {DiCarlo, James J. and Maunsell, John H. R.} }