@booklet {185, title = {How does the primate brain combine generative and discriminative computations in vision?}, year = {2024}, abstract = {

Vision is widely understood as an inference problem. However, two contrasting conceptions of the inference process have each been influential in research on biological vision as well as the engineering of machine vision. The first emphasizes bottom-up signal flow, describing vision as a largely feedforward, discriminative inference process that filters and transforms the visual information to remove irrelevant variation and represent behaviorally relevant information in a format suitable for downstream functions of cognition and behavioral control. In this conception, vision is driven by the sensory data, and perception is direct because the processing proceeds from the data to the latent variables of interest. The notion of \"inference\" in this conception is that of the engineering literature on neural networks, where feedforward convolutional neural networks processing images are said to perform inference. The alternative conception is that of vision as an inference process in Helmholtz\&$\#$39;s sense, where the sensory evidence is evaluated in the context of a generative model of the causal processes giving rise to it. In this conception, vision inverts a generative model through an interrogation of the evidence in a process often thought to involve top-down predictions of sensory data to evaluate the likelihood of alternative hypotheses. The authors include scientists rooted in roughly equal numbers in each of the conceptions and motivated to overcome what might be a false dichotomy between them and engage the other perspective in the realm of theory and experiment. The primate brain employs an unknown algorithm that may combine the advantages of both conceptions. We explain and clarify the terminology, review the key empirical evidence, and propose an empirical research program that transcends the dichotomy and sets the stage for revealing the mysterious hybrid algorithm of primate vision.

}, keywords = {discriminative model, generativemodel, Primatevision, visual inference}, doi = { https://doi.org/10.48550/arXiv.2401.06005}, url = {https://arxiv.org/pdf/2401.06005.pdf}, author = {Benjamin Peters and James J. DiCarlo and Todd Gureckis and Ralf Haefner and Leyla Isik and Joshua Tenenbaum and Talia Konkle and Thomas Naselaris and Kimberly Stachenfeld and Zenna Tavares and Doris Tsao and Ilker Yildirim and Nikolaus Kriegeskorte} } @article {180, title = {Let{\textquoteright}s move forward: Image-computable models and a common model evaluation scheme are prerequisites for a scientific understanding of human visionAbstract}, journal = {Behavioral and Brain Sciences}, volume = {4634}, year = {2023}, month = {Jan-01-2023}, abstract = {

In the target article, Bowers et al. dispute deep artificial neural network (ANN) models as the currently leading models of human vision without producing alternatives. They eschew the use of public benchmarking platforms to compare vision models with the brain and behavior, and they advocate for a fragmented, phenomenon-specific modeling approach. These are unconstructive to scientific progress. We outline how the Brain-Score community is moving forward to add new model-to-human comparisons to its community-transparent suite of benchmarks.

}, issn = {0140-525X}, doi = {10.1017/S0140525X23001607}, url = {https://www.cambridge.org/core/product/identifier/S0140525X23001607/type/journal_article}, author = {DiCarlo, James J. and Yamins, Daniel L. K. and Ferguson, Michael E. and Fedorenko, Evelina and Bethge, Matthias and Bonnen, Tyler and Schrimpf, Martin} } @article {177, title = {A Unifying Principle for the Functional Organization of Visual Cortex}, journal = {bioRxiv}, year = {2023}, month = {2023}, abstract = {

A key feature of many cortical systems is functional organization: the arrangement of neurons with specific functional properties in characteristic spatial patterns across the cortical surface. However, the principles\ underlying the emergence and utility of functional organization are poorly understood. Here we develop\ the Topographic Deep Artificial Neural Network (TDANN), the first unified model to accurately predict the\ functional organization of multiple cortical areas in the primate visual system. We analyze the key factorsresponsible for the TDANN\’s success and find that it strikes a balance between two specific objectives:achieving a task-general sensory representation that is self-supervised, and maximizing the smoothness of\ responses across the cortical sheet according to a metric that scales relative to cortical surface area. In\ turn, the representations learned by the TDANN are lower dimensional and more brain-like than those in\ models that lack a spatial smoothness constraint. Finally, we provide evidence that the TDANN\’s functionalorganization balances performance with inter-area connection length, and use the resulting models for\ a proof-of-principle optimization of cortical prosthetic design. Our results thus offer a unified principle\ for understanding functional organization and a novel view of the functional role of the visual system in\ particular.

}, doi = { https://doi.org/10.1101/2023.05.18.541361}, url = {https://www.biorxiv.org/content/10.1101/2023.05.18.541361v1.full.pdf}, author = {Margalit, Eshed and Lee, Hyodong and Finzi, Dawn and DiCarlo, James J and Grill-Spector, Kalanit and Yamins, Daniel LK} } @article {158, title = {The ThreeDWorld Transport Challenge: A Visually Guided Task-and-Motion Planning Benchmark for Physically Realistic Embodied AI}, journal = {arXiv}, year = {2021}, month = {03/25/2021}, type = {preprint}, abstract = {

We introduce a visually-guided and physics-driven taskand-motion planning benchmark, which we call the ThreeDWorld Transport Challenge. In this challenge, an embodied agent equipped with two 9-DOF articulated arms is spawned randomly in a simulated physical home environment. The agent is required to find a small set of objects scattered around the house, pick them up, and transport them to a desired final location. We also position containers around the house that can be used as tools to assist with transporting objects efficiently. To complete the task, an embodied agent must plan a sequence of actions to change the state of a large number of objects in the face of realistic physical constraints. We build this benchmark challenge using the ThreeDWorld simulation: a virtual 3D environment where all objects respond to physics, and where can be controlled using a fully physics-driven navigation and interaction API. We evaluate several existing agents on this benchmark. Experimental results suggest that: 1) a pure RL model struggles on this challenge; 2) hierarchical planningbased agents can transport some objects but still far from solving this task. We anticipate that this benchmark will empower researchers to develop more intelligent physicsdriven robots for the physical world.\ 

}, doi = {arXiv:2103.14025}, url = {https://arxiv.org/abs/2103.14025}, author = {Gan, Chuang and Zhou, Siyuan and Schwartz, Jeremy and Alter, Seth and Bhandwaldar, Abhishek and Gutfreund, Dan and Yamins, Daniel LK and DiCarlo, James J and McDermott, Josh and Torralba, Antonio} } @article {167, title = {Unsupervised neural network models of the ventral visual stream}, journal = {Proceedings of the National Academy of Sciences}, volume = {118}, year = {2021}, month = {Jul-01-2022}, pages = {e2014196118}, abstract = {

Deep neural networks currently provide the best quantitative models of the response patterns of neurons throughout the primate ventral visual stream. However, such networks have remained implausible as a model of the development of the ventral stream, in part because they are trained with supervised methods requiring many more labels than are accessible to infants during development. Here, we report that recent rapid progress in unsupervised learning has largely closed this gap. We find that neural network models learned with deep unsupervised contrastive embedding methods achieve neural prediction accuracy in multiple ventral visual cortical areas that equals or exceeds that of models derived using today\’s best supervised methods and that the mapping of these neural network models\’ hidden layers is neuroanatomically consistent across the ventral stream. Strikingly, we find that these methods produce brain-like representations even when trained solely with real human child developmental data collected from head-mounted cameras, despite the fact that these datasets are noisy and limited. We also find that semisupervised deep contrastive embeddings can leverage small numbers of labeled examples to produce representations with substantially improved error-pattern consistency to human behavior. Taken together, these results illustrate a use of unsupervised learning to provide a quantitative model of a multiarea cortical brain system and present a strong candidate for a biologically plausible computational theory of primate sensory learning.

}, issn = {0027-8424}, doi = {10.1073/pnas.2014196118}, url = {http://www.pnas.org/lookup/doi/10.1073/pnas.2014196118}, author = {Zhuang, Chengxu and Yan, Siming and Nayebi, Aran and Schrimpf, Martin and Frank, Michael C. and DiCarlo, James J. and Yamins, Daniel L. K.} } @proceedings {156, title = {Correlation-based spatial layout of deep neural network features generates ventral stream topography}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2020}, month = {2/28/2020}, publisher = {COSYNE}, address = {Denver, CO}, abstract = {

The primate visual system is organized into functional maps, including pinwheel-like arrangements of orientationtuned neurons in primary visual cortex (V1) and patches of category-selective neurons in higher visual cortex. Recent work has demonstrated that deep convolutional neural networks (DCNNs) trained for object recognition are good descriptors of neural representations throughout the ventral pathway, with early, intermediate, and late cortical brain areas best predicted by corresponding layers of the DCNN. Despite this success, DCNNs have no inherent spatial layout for features at a given retinotopic location, and thus, make no predictions regarding many of the characteristic topographic phenomena observed in the brain beyond retinotopy itself, e.g., pinwheels and patches. Cortical map formation has been modeled using self-organizing maps that leverage principles of wiring-length minimization and local correlations of unit responses to produce topographic structure. However, these methods rely on simplified feature parameterizations that limit their ability to accommodate more realistic descriptions of neuron response properties, especially in higher visual areas. Here, we augment DCNNs by assigning model units spatial positions in a 2D \“cortical sheet\” and introduce a novel algorithm to arrange units so that local response correlations are maximized. Applying this algorithm to a categorization-optimized DCNN, we find that layouts generated from earlier layers recapitulate core features of V1 orientation, spatial frequency, and color preference maps, while those generated from later layers naturally exhibit category-selective clusters. Because this wide range of apparently disparate phenomenology is produced by the same underlying principle, our results suggest that the functional architecture of the visual system can be explained by two fundamental constraints: the need to perform visual tasks and the pressure to minimize biophysical costs such as wiring length. Our framework for spatially mapping DCNNs integrates biophysical and representational phenomenology, allowing a more unified understanding of the visual system\’s functional architecture.

}, url = {http://cosyne.org/cosyne20/Cosyne2020_program_book.pdf}, author = {Eshed Margalit and Hyodong Lee and Tiago Marques and James J. DiCarlo and Daniel L.K. Yamins} } @article {152, title = {An Open Resource for Non-human Primate Optogenetics}, journal = {Neuron}, year = {2020}, month = {January 10, 2020}, type = {NeuroResource}, abstract = {

Optogenetics has revolutionized neuroscience in small laboratory animals, but its effect on animal models more closely related to humans, such as non-human primates (NHPs), has been mixed. To make evidence-based decisions in primate optogenetics, the scientific community would benefit from a centralized database listing all attempts, successful and unsuccessful, of using optogenetics in the primate brain. We contacted members of the community to ask for their contributions to an open science initiative. As of this writing, 45 laboratories around the world contributed more than 1,000 injection experiments, including precise details regarding their methods and outcomes. Of those entries, more than half had not been published. The resource is free for everyone to consult and contribute to on the Open Science Framework website. Here we review some of the insights from this initial release of the database and discuss methodological considerations to improve the success of optogenetic experiments in NHPs.

}, issn = {08966273}, doi = {10.1016/j.neuron.2020.09.027}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0896627320307510}, author = {Tremblay, Sebastien and Acker, Leah and Afraz, Arash and Albaugh, Daniel L. and Amita, Hidetoshi and Andrei, Ariana R. and Angelucci, Alessandra and Aschner, Amir and Balan, Puiu F. and Basso, Michele A. and Benvenuti, Giacomo and Bohlen, Martin O. and Caiola, Michael J. and Calcedo, Roberto and Cavanaugh, James and Chen, Yuzhi and Chen, Spencer and Chernov, Mykyta M. and Clark, Andrew M. and Dai, Ji and Debes, Samantha R. and Deisseroth, Karl and Desimone, Robert and Dragoi, Valentin and Egger, Seth W. and Eldridge, Mark A.G. and El-Nahal, Hala G. and Fabbrini, Francesco and Federer, Frederick and Fetsch, Christopher R. and Fortuna, Michal G. and Friedman, Robert M. and Fujii, Naotaka and Gail, Alexander and Galvan, Adriana and Ghosh, Supriya and Gieselmann, Marc Alwin and Gulli, Roberto A. and Hikosaka, Okihide and Hosseini, Eghbal A. and Hu, Xing and {\"u}er, Janina and Inoue, Ken-ichi and Janz, Roger and Jazayeri, Mehrdad and Jiang, Rundong and Ju, Niansheng and Kar, Kohitij and Klein, Carsten and Kohn, Adam and Komatsu, Misako and Maeda, Kazutaka and Martinez-Trujillo, Julio C. and Matsumoto, Masayuki and Maunsell, John H.R. and Mendoza-Halliday, Diego and Monosov, Ilya E. and Muers, Ross S. and Nurminen, Lauri and Ortiz-Rios, Michael and {\textquoteright}Shea, Daniel J. and Palfi, {\'e}phane and Petkov, Christopher I. and Pojoga, Sorin and Rajalingham, Rishi and Ramakrishnan, Charu and Remington, Evan D. and Revsine, Cambria and Roe, Anna W. and Sabes, Philip N. and Saunders, Richard C. and Scherberger, {\"o}rg and Schmid, Michael C. and Schultz, Wolfram and Seidemann, Eyal and Senova, Yann-Suhan and Shadlen, Michael N. and Sheinberg, David L. and Siu, Caitlin and Smith, Yoland and Solomon, Selina S. and Sommer, Marc A. and Spudich, John L. and Stauffer, William R. and Takada, Masahiko and Tang, Shiming and Thiele, Alexander and Treue, Stefan and Vanduffel, Wim and Vogels, Rufin and Whitmire, Matthew P. and Wichmann, Thomas and Wurtz, Robert H. and Xu, Haoran and Yazdan-Shahmorad, Azadeh and Shenoy, Krishna V. and DiCarlo, James J. and Platt, Michael L.} } @article {148, title = {ThreeDWorld: A Platform for Interactive Multi-Modal Physical Simulation}, journal = {arXiv}, year = {2020}, month = {July 9, 2020}, type = {preprint}, abstract = {

We introduce ThreeDWorld (TDW), a platform for interactive multi-modal physical simulation. With TDW, users can simulate high-fidelity sensory data and physical interactions between mobile agents and objects in a wide variety of rich 3D environments. TDW has several unique properties: 1) realtime near photo-realistic image rendering quality; 2) a library of objects and environments with materials for high-quality rendering, and routines enabling user customization of the asset library; 3) generative procedures for efficiently building classes of new environments 4) high-fidelity audio rendering; 5) believable and realistic physical interactions for a wide variety of material types, including cloths, liquid, and deformable objects; 6) a range of \"avatar\" types that serve as embodiments of AI agents, with the option for user avatar customization; and 7) support for human interactions with VR devices. TDW also provides a rich API enabling multiple agents to interact within a simulation and return a range of sensor and physics data representing the state of the world. We present initial experiments enabled by the platform around emerging research directions in computer vision, machine learning, and cognitive science, including multi-modal physical scene understanding, multi-agent interactions, models that \"learn like a child\", and attention studies in humans and neural networks. The simulation platform will be made publicly available.

}, url = {https://arxiv.org/abs/2007.04954}, author = {Gan, Chuang and Schwartz, Jeremy and Alter, Seth and Schrimpf, Martin and Traer, James and De Freitas, Julian and Kubilius, Jonas and Bhandwaldar, Abhishek and Haber, Nick and Sano, Megumi and Wang, Elias and Mrowca, Damian and Lingelbach, Michael and Curtis, Aidan and Figelis, Kevin and Bear, Daniel M. and Gutfreund, Dan and Cox, David and DiCarlo, James J. and McDermott, Josh and Tenenbaum, Joshua B. and Yamins, Daniel L.K.} } @article {5, title = {Topographic deep artificial neural networks reproduce the hallmarks of the primate inferior temporal cortex face processing network}, journal = {bioRxiv}, year = {2020}, month = {07/2020}, type = {preprint}, abstract = {

A salient characteristic of monkey inferior temporal (IT) cortex is the IT face processing network. Its hallmarks include: \“face neurons\” that respond more to faces than non-face objects, strong spatial clustering of those neurons in foci at each IT anatomical level (\“face patches\”), and the preferential interconnection of those foci. While some deep artificial neural networks (ANNs) are good predictors of IT neuronal responses, including face neurons, they do not explain those face network hallmarks. Here we ask if they might be explained with a simple, metabolically motivated addition to current ANN ventral stream models. Specifically, we designed and successfully trained topographic deep ANNs (TDANNs) to solve real-world visual recognition tasks (as in prior work), but, in addition, we also optimized each network to minimize a proxy for neuronal wiring length within its IT layers. We report that after this dual optimization, the model IT layers of TDANNs reproduce the hallmarks of the IT face network: the presence of face neurons, clusters of face neurons that quantitatively match those found in IT face patches, connectivity between those patches, and the emergence of face viewpoint invariance along the network hierarchy. We find that these phenomena emerge for a range of naturalistic experience, but not for highly unnatural training. Taken together, these results show that the IT face processing network could be a consequence of a basic hierarchical anatomy along the ventral stream, selection pressure on the visual system to accomplish general object categorization, and selection pressure to minimize axonal wiring length.

}, doi = {https://doi.org/10.1101/2020.07.09.185116}, url = {https://www.biorxiv.org/content/10.1101/2020.07.09.185116v1.full.pdf}, author = {Hyodong Lee and Eshed Margalit and Kamila M. Jozwik and Michael A. Cohen and Nancy Kanwisher and Daniel L. K. Yamins and James J. DiCarlo} } @article {150, title = {Unsupervised Neural Network Models of the Ventral Visual Stream}, journal = {bioRxiv}, year = {2020}, month = {June 18, 2020}, type = {preprint}, abstract = {

Deep neural networks currently provide the best quantitative models of the response patterns of neurons throughout the primate ventral visual stream. However, such networks have remained implausible as a model of the development of the ventral stream, in part because they are trained with supervised methods requiring many more labels than are accessible to infants during development. Here, we report that recent rapid progress in unsupervised learning has largely closed this gap. We find that neural network models learned with deep unsupervised contrastive embedding methods achieve neural prediction accuracy in multiple ventral visual cortical areas that equals or exceeds that of models derived using today\’s best supervised methods, and that the mapping of these neural network models\’ hidden layers is neuroanatomically consistent across the ventral stream. Moreover, we find that these methods produce brain-like representations even when trained on noisy and limited data measured from real children\’s developmental experience. We also find that semi-supervised deep contrastive embeddings can leverage small numbers of labelled examples to produce representations with substantially improved error-pattern consistency to human behavior. Taken together, these results suggest that deep contrastive embedding objectives may be a biologically-plausible computational theory of primate visual development.

}, doi = {10.1101/2020.06.16.155556}, url = {https://www.biorxiv.org/content/10.1101/2020.06.16.155556v1.abstract}, author = {Zhuang, Chengxu and Yan, Siming and Nayebi, Aran and Schrimpf, Martin and Frank, Michael and DiCarlo, James J. and Yamins, Daniel L.K.} } @conference {11, title = {Brain-Like Object Recognition with High-Performing Shallow Recurrent ANNs}, booktitle = {Neural Information Processing Systems}, year = {2019}, abstract = {

Deep convolutional artificial neural networks (ANNs) are the leading class of candidate models of the mechanisms of visual processing in the primate ventral stream. While initially inspired by brain anatomy, over the past years, these ANNs have evolved from a simple eight-layer architecture in AlexNet to extremely deep and branching architectures, demonstrating increasingly better object categorization performance, yet bringing into question how brain-like they still are. In particular, typical deep models from the machine learning community are often hard to map onto the brain\&$\#$39;s anatomy due to their vast number of layers and missing biologically-important connections, such as recurrence. Here we demonstrate that better anatomical alignment to the brain and high performance on machine learning as well as neuroscience measures do not have to be in contradiction. We developed CORnet-S, a shallow ANN with four anatomically mapped areas and recurrent connectivity, guided by Brain-Score, a new large-scale composite of neural and behavioral benchmarks for quantifying the functional fidelity of models of the primate ventral visual stream. Despite being significantly shallower than most models, CORnet-S is the top model on Brain-Score and outperforms similarly compact models on ImageNet. Moreover, our extensive analyses of CORnet-S circuitry variants reveal that recurrence is the main predictive factor of both Brain-Score and ImageNet top-1 performance. Finally, we report that the temporal evolution of the CORnet-S \"IT\" neural population resembles the actual monkey IT population dynamics. Taken together, these results establish CORnet-S, a compact, recurrent ANN, as the current best model of the primate ventral visual stream.

}, doi = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns}, url = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns.pdf}, author = {Jonas Kubilius and Martin Schrimpf and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kohitij Kar and Bashivan, Pouya and Jonathan Prescott-Roy and Kailyn Schmidt and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and James J. DiCarlo} } @conference {166, title = {Using Brain-Score to Evaluate and Build Neural Networks for Brain-Like Object Recognition}, booktitle = {Computational and Systems Neuroscience (COSYNE)}, year = {2019}, address = {Denver, CO}, author = {Schrimpf, Martin and Kubilius, Jonas and Hong, Ha and Majaj, Najib and Rajalingham, Rishi and Issa, Elias B and Kar, Kohitij and Ziemba, Corey M and Bashivan, Pouya and Prescott-Roy, Jonathan and Schmidt, Kailyn and Yamins, Daniel LK and DiCarlo, James J} } @article {22, title = {Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?}, journal = {bioRxiv}, year = {2018}, month = {09/2018}, type = {preprint}, abstract = {

The internal representations of early deep artificial neural networks (ANNs) were found to be remarkably similar to the internal neural representations measured experimentally in the primate brain. Here we ask, as deep ANNs have continued to evolve, are they becoming more or less brain-like? ANNs that are most functionally similar to the brain will contain mechanisms that are most like those used by the brain. We therefore developed Brain-Score - a composite of multiple neural and behavioral benchmarks that score any ANN on how similar it is to the brain\&$\#$39;s mechanisms for core object recognition - and we deployed it to evaluate a wide range of state-of-the-art deep ANNs. Using this scoring system, we here report that: (1) DenseNet-169, CORnet-S and ResNet-101 are the most brain-like ANNs. (2) There remains considerable variability in neural and behavioral responses that is not predicted by any ANN, suggesting that no ANN model has yet captured all the relevant mechanisms. (3) Extending prior work, we found that gains in ANN ImageNet performance led to gains on Brain-Score. However, correlation weakened at \>= 70\% top-1 ImageNet performance, suggesting that additional guidance from neuroscience is needed to make further advances in capturing brain mechanisms. (4) We uncovered smaller (i.e. less complex) ANNs that are more brain-like than many of the best-performing ImageNet models, which suggests the opportunity to simplify ANNs to better understand the ventral stream. The scoring system used here is far from complete. However, we propose that evaluating and tracking model-benchmark correspondences through a Brain-Score that is regularly updated with new brain data is an exciting opportunity: experimental benchmarks can be used to guide machine network evolution, and machine networks are mechanistic hypotheses of the brain\&$\#$39;s network and thus drive next experiments. To facilitate both of these, we release Brain-Score.org: a platform that hosts the neural and behavioral benchmarks, where ANNs for visual processing can be submitted to receive a Brain-Score and their rank relative to other models, and where new experimental data can be naturally incorporated.

}, doi = {https://doi.org/10.1101/407007}, url = {https://www.biorxiv.org/content/10.1101/407007v2.full.pdf}, author = {Martin Schrimpf and Kubilius, Jonas and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kar, Kohitij and Bashivan, Pouya and Jonathan Prescott-Roy and Schmidt, Kailyn and Daniel L. K. Yamins and DiCarlo, James J.} } @article {23, title = {CORnet: Modeling the Neural Mechanisms of Core Object Recognition}, journal = {bioRxiv}, year = {2018}, month = {09/2018}, type = {preprint}, abstract = {

Deep artificial neural networks with spatially repeated processing (a.k.a., deep convolutional ANNs) have been established as the best class of candidate models of visual processing in primate ventral visual processing stream. Over the past five years, these ANNs have evolved from a simple feedforward eight-layer architecture in AlexNet to extremely deep and branching NASNet architectures, demonstrating increasingly better object categorization performance and increasingly better explanatory power of both neural and behavioral responses. However, from the neuroscientist\&$\#$39;s point of view, the relationship between such very deep architectures and the ventral visual pathway is incomplete in at least two ways. On the one hand, current state-of-the-art ANNs appear to be too complex (e.g., now over 100 levels) compared with the relatively shallow cortical hierarchy (4-8 levels), which makes it difficult to map their elements to those in the ventral visual stream and to understand what they are doing. On the other hand, current state-of-the-art ANNs appear to be not complex enough in that they lack recurrent connections and the resulting neural response dynamics that are commonplace in the ventral visual stream. Here we describe our ongoing efforts to resolve both of these issues by developing a \"CORnet\" family of deep neural network architectures. Rather than just seeking high object recognition performance (as the state-of-the-art ANNs above), we instead try to reduce the model family to its most important elements and then gradually build new ANNs with recurrent and skip connections while monitoring both performance and the match between each new CORnet model and a large body of primate brain and behavioral data. We report here that our current best ANN model derived from this approach (CORnet-S) is among the top models on Brain-Score, a composite benchmark for comparing models to the brain, but is simpler than other deep ANNs in terms of the number of convolutions performed along the longest path of information processing in the model. All CORnet models are available at\ https://github.com/dicarlolab/CORnet, and we plan to update this manuscript and the available models in this family as they are produced.

}, doi = {https://doi.org/10.1101/408385}, url = {https://www.biorxiv.org/content/10.1101/408385v1.full.pdf}, author = {Kubilius, Jonas and Martin Schrimpf and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and DiCarlo, James J.} } @article {20, title = {Task-Driven Convolutional Recurrent Models of the Visual System}, journal = {arXiv}, year = {2018}, month = {06/2018}, type = {preprint}, abstract = {

Feed-forward convolutional neural networks (CNNs) are currently state-of-the-art for object classification tasks such as ImageNet. Further, they are quantitatively accurate models of temporally-averaged responses of neurons in the primate brain\&$\#$39;s visual system. However, biological visual systems have two ubiquitous architectural features not shared with typical CNNs: local recurrence within cortical areas, and long-range feedback from downstream areas to upstream areas. Here we explored the role of recurrence in improving classification performance. We found that standard forms of recurrence (vanilla RNNs and LSTMs) do not perform well within deep CNNs on the ImageNet task. In contrast, custom cells that incorporated two structural features, bypassing and gating, were able to boost task accuracy substantially. We extended these design principles in an automated search over thousands of model architectures, which identified novel local recurrent cells and long-range feedback connections useful for object recognition. Moreover, these task-optimized ConvRNNs explained the dynamics of neural activity in the primate visual system better than feedforward networks, suggesting a role for the brain\&$\#$39;s recurrent connections in performing difficult visual behaviors.

}, doi = {https://arxiv.org/abs/1807.00053}, url = {https://arxiv.org/pdf/1807.00053.pdf}, author = {Aran Nayebi and Daniel Bear and Kubilius, Jonas and Ganguli, S and Sussillo, D and DiCarlo, James J. and Yamins, DLK} } @article {25, title = {Eight open questions in the computational modeling of higher sensory cortex}, journal = {Current Opinion in Neurobiology}, volume = {37}, year = {2016}, month = {01/2016}, pages = {114 - 120}, abstract = {

Propelled by advances in biologically inspired computer vision and artificial intelligence, the past five years have seen significant progress in using deep neural networks to model response patterns of neurons in visual cortex. In this paper, we briefly review this progress and then discuss eight key \&$\#$39;open questions\&$\#$39; that we believe will drive research in computational models of sensory systems over the next five years, both in visual cortex and beyond.

}, issn = {1873-6882}, doi = {10.1016/j.conb.2016.02.001}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0959438816300022}, author = {Yamins, Daniel LK and DiCarlo, James J} } @article {27, title = {Explicit information for category-orthogonal object properties increases along the ventral stream}, journal = {Nature Neuroscience}, volume = {19}, year = {2016}, month = {02/2016}, pages = {613 - 622}, abstract = {

Extensive research has revealed that the ventral visual stream hierarchically builds a robust representation for supporting visual object categorization tasks. We systematically explored the ability of multiple ventral visual areas to support a variety of \&$\#$39;category-orthogonal\&$\#$39; object properties such as position, size and pose. For complex naturalistic stimuli, we found that the inferior temporal (IT) population encodes all measured category-orthogonal object properties, including those properties often considered to be low-level features (for example, position), more explicitly than earlier ventral stream areas. We also found that the IT population better predicts human performance patterns across properties. A hierarchical neural network model based on simple computational principles generates these same cross-area patterns of information. Taken together, our empirical results support the hypothesis that all behaviorally relevant object properties are extracted in concert up the ventral visual hierarchy, and our computational model explains how that hierarchy might be built.

}, issn = {1097-6256}, doi = {10.1038/nn.4247}, url = {http://www.nature.com/articles/nn.4247}, author = {Hong, Ha and Yamins, Daniel L K and Majaj, Najib J and DiCarlo, James J} } @article {26, title = {Using goal-driven deep learning models to understand sensory cortex}, journal = {Nature Neuroscience}, volume = {19}, year = {2016}, month = {01/2016}, pages = {356 - 365}, abstract = {

Fueled by innovation in the computer vision and artificial intelligence communities, recent developments in computational neuroscience have used goal-driven hierarchical convolutional neural networks (HCNNs) to make strides in modeling neural single-unit and population responses in higher visual cortical areas. In this Perspective, we review the recent progress in a broader modeling context and describe some of the key technical innovations that have supported it. We then outline how the goal-driven HCNN approach can be used to delve even more deeply into understanding the development and organization of sensory cortical processing.

}, issn = {1097-6256}, doi = {10.1038/nn.4244}, url = {http://www.nature.com/articles/nn.4244.pdf}, author = {Yamins, Daniel L K and DiCarlo, James J} } @article {32, title = {Deep Neural Networks Rival the Representation of Primate IT Cortex for Core Visual Object Recognition}, journal = {PLoS Computational Biology}, volume = {10}, year = {2014}, month = {12/2014}, pages = {e1003963}, abstract = {

The primate visual system achieves remarkable visual object recognition performance even in brief presentations, and under changes to object exemplar, geometric transformations, and background variation (a.k.a. core visual object recognition). This remarkable performance is mediated by the representation formed in inferior temporal (IT) cortex. In parallel, recent advances in machine learning have led to ever higher performing models of object recognition using artificial deep neural networks (DNNs). It remains unclear, however, whether the representational performance of DNNs rivals that of the brain. To accurately produce such a comparison, a major difficulty has been a unifying metric that accounts for experimental limitations, such as the amount of noise, the number of neural recording sites, and the number of trials, and computational limitations, such as the complexity of the decoding classifier and the number of classifier training examples. In this work, we perform a direct comparison that corrects for these experimental limitations and computational considerations. As part of our methodology, we propose an extension of \"kernel analysis\" that measures the generalization accuracy as a function of representational complexity. Our evaluations show that, unlike previous bio-inspired models, the latest DNNs rival the representational performance of IT cortex on this visual object recognition task. Furthermore, we show that models that perform well on measures of representational performance also perform well on measures of representational similarity to IT, and on measures of predicting individual IT multi-unit responses. Whether these DNNs rely on computational mechanisms similar to the primate visual system is yet to be determined, but, unlike all previous bio-inspired models, that possibility cannot be ruled out merely on representational performance grounds.

}, doi = {10.1371/journal.pcbi.1003963}, url = {https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003963\&type=printable}, author = {Cadieu, Charles F. and Hong, Ha and Yamins, Daniel L. K. and Pinto, Nicolas and Ardila, Diego and Solomon, Ethan A. and Majaj, Najib J. and DiCarlo, James J.}, editor = {Bethge, Matthias} } @article {33, title = {Neural Mechanisms Underlying Visual Object Recognition}, journal = {Cold Spring Harbor Symposia on Quantitative Biology}, volume = {79}, year = {2014}, month = {2014}, pages = {99 - 107}, abstract = {

Invariant visual object recognition and the underlying neural representations are fundamental to higher-level human cognition. To understand these neural underpinnings, we combine human and monkey psychophysics, large-scale neurophysiology, neural perturbation methods, and computational modeling to construct falsifiable, predictive models that aim to fully account for the neural encoding and decoding processes that underlie visual object recognition. A predictive encoding model must minimally describe the transformation of the retinal image to population patterns of neural activity along the entire cortical ventral stream of visual processing and must accurately predict the responses to any retinal image. A predictive decoding model must minimally describe the transformation from those population patterns of neural activity to observed object recognition behavior (i.e., subject reports), and, given that population pattern of activity, it must accurately predict behavior for any object recognition task. To date, we have focused on core object recognition-a remarkable behavior that is accomplished with image viewing durations of \<200 msec. Our work thus far reveals that the neural encoding process is reasonably well explained by a largely feed-forward, highly complex, multistaged nonlinear neural network-the current best neuronal simulation models predict approximately one-half of the relevant neuronal response variance across the highest levels of the ventral stream (areas V4 and IT). Remarkably, however, the decoding process from IT to behavior for all object recognition tasks tested thus far is very accurately predicted by simple direct linear conversion of the inferior temporal neural population state to behavior choice. We have recently examined the behavioral consequences of direct suppression of IT neural activity using pharmacological and optogenetic methods and find them to be well-explained by the same linear decoding model.

}, issn = {0091-7451}, doi = {10.1101/sqb.2014.79.024729}, url = {http://symposium.cshlp.org/content/79/99.full.pdf+html}, author = {Afraz, Arash and Yamins, Daniel L.K. and DiCarlo, James J.} } @article {31, title = {Performance-optimized hierarchical models predict neural responses in higher visual cortex}, journal = {Proceedings of the National Academy of Sciences}, year = {2014}, month = {10/2014}, pages = {8619 - 8624}, abstract = {

The ventral visual stream underlies key human visual object recognition abilities. However, neural encoding in the higher areas of the ventral stream remains poorly understood. Here, we describe a modeling approach that yields a quantitatively accurate model of inferior temporal (IT) cortex, the highest ventral cortical area. Using high-throughput computational techniques, we discovered that, within a class of biologically plausible hierarchical neural network models, there is a strong correlation between a model\’s categorization performance and its ability to predict individual IT neural unit response data. To pursue this idea, we then identified a high-performing neural network that matches human performance on a range of recognition tasks. Critically, even though we did not constrain this model to match neural data, its top output layer turns out to be highly predictive of IT spiking responses to complex naturalistic images at both the single site and population levels. Moreover, the model\’s intermediate layers are highly predictive of neural responses in the V4 cortex, a midlevel visual area that provides the dominant cortical input to IT. These results show that performance optimization\—applied in a biologically appropriate model class\—can be used to build quantitative predictive models of neural processing.

}, issn = {0027-8424}, doi = {10.1073/pnas.1403112111}, url = {http://www.pnas.org/cgi/doi/10.1073/pnas.1403112111}, author = {Yamins, D. L. K. and Hong, H. and Cadieu, C. F. and Solomon, E. A. and Seibert, D. and DiCarlo, J. J.} } @conference {37, title = {Hierarchical Modular Optimization of Convolutional Networks Achieves Representations Similar to Macaque IT and Human Ventral Stream}, booktitle = {Advances in Neural Information Processing Systems}, year = {2013}, month = {12/2013}, address = {Lake Tahoe, Nevada, United States.}, abstract = {

Humans recognize visually-presented objects rapidly and accurately. To understand this ability, we seek to construct models of the ventral stream, the series of cortical areas thought to subserve object recognition. One tool to assess the quality of a model of the ventral stream is the Representational Dissimilarity Matrix (RDM), which uses a set of visual stimuli and measures the distances produced in either the brain (i.e. fMRI voxel responses, neural firing rates) or in models (features). Previous work has shown that all known models of the ventral stream fail to capture the RDM pattern observed in either IT cortex, the highest ventral area, or in the human ventral stream. In this work, we construct models of the ventral stream using a novel optimization procedure for category-level object recognition problems, and produce RDMs resembling both macaque IT and human ventral stream. The model, while novel in the optimization procedure, further develops a long-standing functional hypothesis that the ventral visual stream is a hierarchically arranged series of processing stages optimized for visual object recognition.

}, doi = {https://papers.nips.cc/paper/4991-hierarchical-modular-optimization-of-convolutional-networks-achieves-representations-similar-to-macaque-it-and-human-ventral-stream}, url = {https://papers.nips.cc/paper/4991-hierarchical-modular-optimization-of-convolutional-networks-achieves-representations-similar-to-macaque-it-and-human-ventral-stream.pdf}, author = {Daniel L. K. Yamins and Ha Hong and Cadieu, Charles F and DiCarlo, James J.} } @article {36, title = {The Neural Representation Benchmark and its Evaluation on Brain and Machine}, journal = {arXiv}, year = {2013}, month = {01/2013}, type = {preprint}, abstract = {

A key requirement for the development of effective learning representations is their evaluation and comparison to representations we know to be effective. In natural sensory domains, the community has viewed the brain as a source of inspiration and as an implicit benchmark for success. However, it has not been possible to directly test representational learning algorithms directly against the representations contained in neural systems. Here, we propose a new benchmark for visual representations on which we have directly tested the neural representation in multiple visual cortical areas in macaque (utilizing data from [Majaj et al., 2012]), and on which any computer vision algorithm that produces a feature space can be tested. The benchmark measures the effectiveness of the neural or machine representation by computing the classification loss on the ordered eigendecomposition of a kernel matrix [Montavon et al., 2011]. In our analysis we find that the neural representation in visual area IT is superior to visual area V4. In our analysis of representational learning algorithms, we find that three-layer models approach the representational performance of V4 and the algorithm in [Le et al., 2012] surpasses the performance of V4. Impressively, we find that a recent supervised algorithm [Krizhevsky et al., 2012] achieves performance comparable to that of IT for an intermediate level of image variation difficulty, and surpasses IT at a higher difficulty level. We believe this result represents a major milestone: it is the first learning algorithm we have found that exceeds our current estimate of IT representation performance. We hope that this benchmark will assist the community in matching the representational performance of visual cortex and will serve as an initial rallying point for further correspondence between representations derived in brains and machines.

}, doi = {https://arxiv.org/abs/1301.3530}, url = {https://arxiv.org/pdf/1301.3530.pdf}, author = {Cadieu, Charles F. and Ha Hong and Daniel L. K. Yamins and Pinto, Nicolas and Majaj, Najib J and James J. DiCarlo} }