@conference {187, title = {Learning only a handful of latent variables produces neural-aligned CNN models of the ventral stream}, booktitle = {Computational and Systems Neuroscience (COSYNE) }, year = {2024}, address = {Lisbon, Portugal}, abstract = {

Image-computable modeling of primate ventral stream visual processing has made great strides via brainmapped versions of convolutional neural networks (CNNs) that are optimized on thousands of object categories (ImageNet), the performance of which strongly predicts CNNs\’ neural alignment. However, human and primate visual intelligence extends far beyond object categorization, encompassing a diverse range of tasks, such as estimating the latent variables of object position or pose in the image. The influence of task choice on neural alignment in CNNs, compared to CNN architecture, remains underexplored, partly due to the scarcity of largescale datasets with rich known labels beyond categories. 3D graphic engines, capable of creating training images with detailed information on various latent variables, offer a solution. Here, we asked how the choice of visual tasks that are used to train CNNs (i.e., the set of latent variables to be estimated) affects their ventral stream neural alignment. We focused on the estimation of variables such as object position and pose, and we tested CNNs\’ neural alignment via the Brain-Score open science platform. We found some of these CNNs had neural alignment scores that were very close to those trained on ImageNet, even though their entire training experience has been on synthetic images. Additionally, we found training models on just a handful of latent variables achieved the same level of neural alignment as models trained on a much larger number of categories, suggesting that latent variable training is more efficient than category training in driving model-neural alignment. Moreover, we found that these models\’ neural alignment scores scale with the amount of synthetic data used during training, suggesting the potential of obtaining more aligned models with larger synthetic datasets. This study highlights the effectiveness of using synthetic datasets and latent variables in advancing image-computable models of the ventral visual stream.

}, url = {https://hdl.handle.net/1721.1/153744}, author = {Xie, Yudi and Alter, Esther and Schwartz, Jeremy and DiCarlo, James J} } @article {178, title = {fROI-level computational models enable broad-scale experimental testing and expose key divergences between models and brains}, journal = {Journal of Vision}, volume = {23}, year = {2023}, month = {2023}, pages = {5788 - 5788}, abstract = {

Deep convolutional neural network (DNN)- based models have emerged as our leading hypotheses of human vision. Here we describe, and expand upon, our latest effort to use DNN models of brain regions to explain key results from previous cognitive neuroscience and psychology experiments. Many stimuli in these prior experiments were highly manipulated (e.g. scrambled body parts, face parts, re-arranged spatial positions) often outside the domain of natural stimuli. These results can therefore be considered as tests of model generalization beyond naturalistic stimuli. We first performed these tests on the fusiform face area (FFA), parahippocampal place area (PPA) and the extrastriate body area (EBA). Our previous results (presented in VSS2022) showed that our fROI-level models recapitulate several key results from prior studies. We also observed that models did not perform as well on non-naturalistic stimuli. Here we extend our model evaluation metrics in two ways. First, we replicated findings from the original paper on the EBA (Downing et al 2001) that the EBA responds as highly to line drawings of bodies, and symbolic stick figures as to natural images of bodies (and not to control conditions like faces and objects). Second, we find that none of the computational models explain this pattern of observed responses, though models trained with language-based supervision (like CLIP) do better than other models. Together, our results on symbolic body images expose the bounds of current computational models. This progress was made possible only because of fROI-level modeling procedures, and opens up new ways to understand the power and limitations of current models and test novel hypotheses completely in-silico.

}, isbn = {1534-7362}, url = {https://doi.org/10.1167/jov.23.9.5788}, author = {Mieczkowski, Elizabeth and Abate, Alex and De Faria, Willian and Lydic, Kirsten and DiCarlo, James and Kanwisher, Nancy and Apurva Ratan Murty, N.} } @article {162, title = {Chronically implantable LED arrays for behavioral optogenetics in primates}, journal = {Nature Methods}, volume = {18}, year = {2021}, month = {Jan-09-2021}, pages = {1112 - 1116}, abstract = {

Optogenetic methods have been widely used in rodent brains, but remain relatively under-developed for nonhuman primates such as rhesus macaques, an animal model with a large brain expressing sophisticated sensory, motor and cognitive behaviors. To address challenges in behavioral optogenetics in large brains, we developed Opto-Array, a chronically implantable array of light-emitting diodes for high-throughput optogenetic perturbation. We demonstrated that optogenetic silencing in the macaque primary visual cortex with the help of the Opto-Array results in reliable retinotopic visual deficits in a luminance discrimination task. We separately confirmed that Opto-Array illumination results in local neural silencing, and that behavioral effects are not due to tissue heating. These results demonstrate the effectiveness of the Opto-Array for behavioral optogenetic applications in large brains.

}, issn = {1548-7091}, doi = {10.1038/s41592-021-01238-9}, url = {https://www.nature.com/articles/s41592-021-01238-9}, author = {Rajalingham, Rishi and Sorenson, Michael and Azadi, Reza and Bohn, Simon and DiCarlo, James J. and Afraz, Arash} } @article {161, title = {Computational models of category-selective brain regions enable high-throughput tests of selectivity}, journal = {Nature Communications}, volume = {12}, year = {2021}, month = {Jan-12-2021}, abstract = {

Cortical regions apparently selective to faces, places, and bodies have provided important evidence for domain-specific theories of human cognition, development, and evolution. But claims of category selectivity are not quantitatively precise and remain vulnerable to empirical refutation. Here we develop artificial neural network-based encoding models that accurately predict the response to novel images in the fusiform face area, parahippocampal place area, and extrastriate body area, outperforming descriptive models and experts. We use these models to subject claims of category selectivity to strong tests, by screening for and synthesizing images predicted to produce high responses. We find that these high-responsepredicted images are all unambiguous members of the hypothesized preferred category for each region. These results provide accurate, image-computable encoding models of each category-selective region, strengthen evidence for domain specificity in the brain, and point the way for future research characterizing the functional organization of the brain with unprecedented computational precision.

}, doi = {10.1038/s41467-021-25409-6}, url = {https://www.nature.com/articles/s41467-021-25409-6}, author = {Murty, NAR and Bashivan, Pouya and Abate, Alex and DiCarlo, James J. and Kanwisher, Nancy} } @article {158, title = {The ThreeDWorld Transport Challenge: A Visually Guided Task-and-Motion Planning Benchmark for Physically Realistic Embodied AI}, journal = {arXiv}, year = {2021}, month = {03/25/2021}, type = {preprint}, abstract = {

We introduce a visually-guided and physics-driven taskand-motion planning benchmark, which we call the ThreeDWorld Transport Challenge. In this challenge, an embodied agent equipped with two 9-DOF articulated arms is spawned randomly in a simulated physical home environment. The agent is required to find a small set of objects scattered around the house, pick them up, and transport them to a desired final location. We also position containers around the house that can be used as tools to assist with transporting objects efficiently. To complete the task, an embodied agent must plan a sequence of actions to change the state of a large number of objects in the face of realistic physical constraints. We build this benchmark challenge using the ThreeDWorld simulation: a virtual 3D environment where all objects respond to physics, and where can be controlled using a fully physics-driven navigation and interaction API. We evaluate several existing agents on this benchmark. Experimental results suggest that: 1) a pure RL model struggles on this challenge; 2) hierarchical planningbased agents can transport some objects but still far from solving this task. We anticipate that this benchmark will empower researchers to develop more intelligent physicsdriven robots for the physical world.\ 

}, doi = {arXiv:2103.14025}, url = {https://arxiv.org/abs/2103.14025}, author = {Gan, Chuang and Zhou, Siyuan and Schwartz, Jeremy and Alter, Seth and Bhandwaldar, Abhishek and Gutfreund, Dan and Yamins, Daniel LK and DiCarlo, James J and McDermott, Josh and Torralba, Antonio} } @article {149, title = {Chronically implantable LED arrays for behavioral optogenetics in primates}, journal = {bioRxiv}, year = {2020}, month = {9/11/2020}, type = {preprint}, abstract = {

Challenges in behavioral optogenetics in large brains demand development of a chronically implantable platform for light delivery. We have developed Opto-Array, a chronically implantable array of LEDs for high-throughput optogenetic perturbation in non-human primates. We tested the Opto-Array in the primary visual cortex of a macaque monkey, and demonstrated that optogenetic cortical silencing by the Opto-Array results in reliable retinotopic visual deficits on a luminance discrimination task.

}, doi = {10.1101/2020.09.10.291583}, url = {https://www.biorxiv.org/content/10.1101/2020.09.10.291583v1.abstract}, author = {Rajalingham, Rish and Sorenson, Michael and Azadi, Reza and Bohn, Simon and DiCarlo, James J. and Afraz, Arash} } @article {147, title = {Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence}, journal = {Neuron}, year = {2020}, month = {Jan-09-2020}, abstract = {

A potentially organizing goal of the brain and cognitive sciences is to accurately explain domains of human intelligence as executable, neurally mechanistic models. Years of research have led to models that capture experimental results in individual behavioral tasks and individual brain regions. We here advocate for taking the next step: integrating experimental results from many laboratories into suites of benchmarks that, when considered together, push mechanistic models toward explaining entire domains of intelligence, such as vision, language, and motor control. Given recent successes of neurally mechanistic models and the surging availability of neural, anatomical, and behavioral data, we believe that now is the time to create integrative benchmarking platforms that incentivize ambitious, unified models. This perspective discusses the advantages and the challenges of this approach and proposes specific steps to achieve this goal in the domain of visual intelligence with the case study of an integrative benchmarking platform called Brain-Score.

}, issn = {08966273}, doi = {10.1016/j.neuron.2020.07.040}, url = {https://linkinghub.elsevier.com/retrieve/pii/S089662732030605X}, author = {Schrimpf, Martin and Kubilius, Jonas and Lee, Michael J. and Murty, NAR and Ajemian, Robert and DiCarlo, James J.} } @article {152, title = {An Open Resource for Non-human Primate Optogenetics}, journal = {Neuron}, year = {2020}, month = {January 10, 2020}, type = {NeuroResource}, abstract = {

Optogenetics has revolutionized neuroscience in small laboratory animals, but its effect on animal models more closely related to humans, such as non-human primates (NHPs), has been mixed. To make evidence-based decisions in primate optogenetics, the scientific community would benefit from a centralized database listing all attempts, successful and unsuccessful, of using optogenetics in the primate brain. We contacted members of the community to ask for their contributions to an open science initiative. As of this writing, 45 laboratories around the world contributed more than 1,000 injection experiments, including precise details regarding their methods and outcomes. Of those entries, more than half had not been published. The resource is free for everyone to consult and contribute to on the Open Science Framework website. Here we review some of the insights from this initial release of the database and discuss methodological considerations to improve the success of optogenetic experiments in NHPs.

}, issn = {08966273}, doi = {10.1016/j.neuron.2020.09.027}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0896627320307510}, author = {Tremblay, Sebastien and Acker, Leah and Afraz, Arash and Albaugh, Daniel L. and Amita, Hidetoshi and Andrei, Ariana R. and Angelucci, Alessandra and Aschner, Amir and Balan, Puiu F. and Basso, Michele A. and Benvenuti, Giacomo and Bohlen, Martin O. and Caiola, Michael J. and Calcedo, Roberto and Cavanaugh, James and Chen, Yuzhi and Chen, Spencer and Chernov, Mykyta M. and Clark, Andrew M. and Dai, Ji and Debes, Samantha R. and Deisseroth, Karl and Desimone, Robert and Dragoi, Valentin and Egger, Seth W. and Eldridge, Mark A.G. and El-Nahal, Hala G. and Fabbrini, Francesco and Federer, Frederick and Fetsch, Christopher R. and Fortuna, Michal G. and Friedman, Robert M. and Fujii, Naotaka and Gail, Alexander and Galvan, Adriana and Ghosh, Supriya and Gieselmann, Marc Alwin and Gulli, Roberto A. and Hikosaka, Okihide and Hosseini, Eghbal A. and Hu, Xing and {\"u}er, Janina and Inoue, Ken-ichi and Janz, Roger and Jazayeri, Mehrdad and Jiang, Rundong and Ju, Niansheng and Kar, Kohitij and Klein, Carsten and Kohn, Adam and Komatsu, Misako and Maeda, Kazutaka and Martinez-Trujillo, Julio C. and Matsumoto, Masayuki and Maunsell, John H.R. and Mendoza-Halliday, Diego and Monosov, Ilya E. and Muers, Ross S. and Nurminen, Lauri and Ortiz-Rios, Michael and {\textquoteright}Shea, Daniel J. and Palfi, {\'e}phane and Petkov, Christopher I. and Pojoga, Sorin and Rajalingham, Rishi and Ramakrishnan, Charu and Remington, Evan D. and Revsine, Cambria and Roe, Anna W. and Sabes, Philip N. and Saunders, Richard C. and Scherberger, {\"o}rg and Schmid, Michael C. and Schultz, Wolfram and Seidemann, Eyal and Senova, Yann-Suhan and Shadlen, Michael N. and Sheinberg, David L. and Siu, Caitlin and Smith, Yoland and Solomon, Selina S. and Sommer, Marc A. and Spudich, John L. and Stauffer, William R. and Takada, Masahiko and Tang, Shiming and Thiele, Alexander and Treue, Stefan and Vanduffel, Wim and Vogels, Rufin and Whitmire, Matthew P. and Wichmann, Thomas and Wurtz, Robert H. and Xu, Haoran and Yazdan-Shahmorad, Azadeh and Shenoy, Krishna V. and DiCarlo, James J. and Platt, Michael L.} } @article {148, title = {ThreeDWorld: A Platform for Interactive Multi-Modal Physical Simulation}, journal = {arXiv}, year = {2020}, month = {July 9, 2020}, type = {preprint}, abstract = {

We introduce ThreeDWorld (TDW), a platform for interactive multi-modal physical simulation. With TDW, users can simulate high-fidelity sensory data and physical interactions between mobile agents and objects in a wide variety of rich 3D environments. TDW has several unique properties: 1) realtime near photo-realistic image rendering quality; 2) a library of objects and environments with materials for high-quality rendering, and routines enabling user customization of the asset library; 3) generative procedures for efficiently building classes of new environments 4) high-fidelity audio rendering; 5) believable and realistic physical interactions for a wide variety of material types, including cloths, liquid, and deformable objects; 6) a range of \"avatar\" types that serve as embodiments of AI agents, with the option for user avatar customization; and 7) support for human interactions with VR devices. TDW also provides a rich API enabling multiple agents to interact within a simulation and return a range of sensor and physics data representing the state of the world. We present initial experiments enabled by the platform around emerging research directions in computer vision, machine learning, and cognitive science, including multi-modal physical scene understanding, multi-agent interactions, models that \"learn like a child\", and attention studies in humans and neural networks. The simulation platform will be made publicly available.

}, url = {https://arxiv.org/abs/2007.04954}, author = {Gan, Chuang and Schwartz, Jeremy and Alter, Seth and Schrimpf, Martin and Traer, James and De Freitas, Julian and Kubilius, Jonas and Bhandwaldar, Abhishek and Haber, Nick and Sano, Megumi and Wang, Elias and Mrowca, Damian and Lingelbach, Michael and Curtis, Aidan and Figelis, Kevin and Bear, Daniel M. and Gutfreund, Dan and Cox, David and DiCarlo, James J. and McDermott, Josh and Tenenbaum, Joshua B. and Yamins, Daniel L.K.} } @article {24, title = {Neurophysiological Organization of the Middle Face Patch in Macaque Inferior Temporal Cortex}, journal = {The Journal of Neuroscience}, volume = {36}, year = {2016}, month = {02/2017}, pages = {12729 - 12745}, abstract = {

While early cortical visual areas contain fine scale spatial organization of neuronal properties, such as orientation preference, the spatial organization of higher-level visual areas is less well understood. The fMRI demonstration of face-preferring regions in human ventral cortex and monkey inferior temporal cortex (\“face patches\”) raises the question of how neural selectivity for faces is organized. Here, we targeted hundreds of spatially registered neural recordings to the largest fMRI-identified face-preferring region in monkeys, the middle face patch (MFP), and show that the MFP contains a graded enrichment of face-preferring neurons. At its center, as much as 93\% of the sites we sampled responded twice as strongly to faces than to nonface objects. We estimate the maximum neurophysiological size of the MFP to be \∼6 mm in diameter, consistent with its previously reported size under fMRI. Importantly, face selectivity in the MFP varied strongly even between neighboring sites. Additionally, extremely face-selective sites were \∼40 times more likely to be present inside the MFP than outside. These results provide the first direct quantification of the size and neural composition of the MFP by showing that the cortical tissue localized to the fMRI defined region consists of a very high fraction of face-preferring sites near its center, and a monotonic decrease in that fraction along any radial spatial axis.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0237-16.2016}, url = {https://www.jneurosci.org/content/jneuro/36/50/12729.full.pdf}, author = {Aparicio, Paul L. and Issa, Elias B. and DiCarlo, James J.} } @article {30, title = {Optogenetic and pharmacological suppression of spatial clusters of face neurons reveal their causal role in face gender discrimination}, journal = {Proceedings of the National Academy of Sciences}, year = {2015}, month = {05/2015}, pages = {6730 - 6735}, abstract = {

Neurons that respond more to images of faces over nonface objects were identified in the inferior temporal (IT) cortex of primates three decades ago. Although it is hypothesized that perceptual discrimination between faces depends on the neural activity of IT subregions enriched with \“face neurons,\” such a causal link has not been directly established. Here, using optogenetic and pharmacological methods, we reversibly suppressed the neural activity in small subregions of IT cortex of macaque monkeys performing a facial gender-discrimination task. Each type of intervention independently demonstrated that suppression of IT subregions enriched in face neurons induced a contralateral deficit in face gender-discrimination behavior. The same neural suppression of other IT subregions produced no detectable change in behavior. These results establish a causal link between the neural activity in IT face neuron subregions and face gender-discrimination behavior. Also, the demonstration that brief neural suppression of specific spatial subregions of IT induces behavioral effects opens the door for applying the technical advantages of optogenetics to a systematic attack on the causal relationship between IT cortex and high-level visual perception.

}, issn = {0027-8424}, doi = {10.1073/pnas.1423328112}, url = {http://www.pnas.org/lookup/doi/10.1073/pnas.1423328112}, author = {Afraz, Arash and Boyden, Edward S. and DiCarlo, James J.} } @article {32, title = {Deep Neural Networks Rival the Representation of Primate IT Cortex for Core Visual Object Recognition}, journal = {PLoS Computational Biology}, volume = {10}, year = {2014}, month = {12/2014}, pages = {e1003963}, abstract = {

The primate visual system achieves remarkable visual object recognition performance even in brief presentations, and under changes to object exemplar, geometric transformations, and background variation (a.k.a. core visual object recognition). This remarkable performance is mediated by the representation formed in inferior temporal (IT) cortex. In parallel, recent advances in machine learning have led to ever higher performing models of object recognition using artificial deep neural networks (DNNs). It remains unclear, however, whether the representational performance of DNNs rivals that of the brain. To accurately produce such a comparison, a major difficulty has been a unifying metric that accounts for experimental limitations, such as the amount of noise, the number of neural recording sites, and the number of trials, and computational limitations, such as the complexity of the decoding classifier and the number of classifier training examples. In this work, we perform a direct comparison that corrects for these experimental limitations and computational considerations. As part of our methodology, we propose an extension of \"kernel analysis\" that measures the generalization accuracy as a function of representational complexity. Our evaluations show that, unlike previous bio-inspired models, the latest DNNs rival the representational performance of IT cortex on this visual object recognition task. Furthermore, we show that models that perform well on measures of representational performance also perform well on measures of representational similarity to IT, and on measures of predicting individual IT multi-unit responses. Whether these DNNs rely on computational mechanisms similar to the primate visual system is yet to be determined, but, unlike all previous bio-inspired models, that possibility cannot be ruled out merely on representational performance grounds.

}, doi = {10.1371/journal.pcbi.1003963}, url = {https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003963\&type=printable}, author = {Cadieu, Charles F. and Hong, Ha and Yamins, Daniel L. K. and Pinto, Nicolas and Ardila, Diego and Solomon, Ethan A. and Majaj, Najib J. and DiCarlo, James J.}, editor = {Bethge, Matthias} } @article {33, title = {Neural Mechanisms Underlying Visual Object Recognition}, journal = {Cold Spring Harbor Symposia on Quantitative Biology}, volume = {79}, year = {2014}, month = {2014}, pages = {99 - 107}, abstract = {

Invariant visual object recognition and the underlying neural representations are fundamental to higher-level human cognition. To understand these neural underpinnings, we combine human and monkey psychophysics, large-scale neurophysiology, neural perturbation methods, and computational modeling to construct falsifiable, predictive models that aim to fully account for the neural encoding and decoding processes that underlie visual object recognition. A predictive encoding model must minimally describe the transformation of the retinal image to population patterns of neural activity along the entire cortical ventral stream of visual processing and must accurately predict the responses to any retinal image. A predictive decoding model must minimally describe the transformation from those population patterns of neural activity to observed object recognition behavior (i.e., subject reports), and, given that population pattern of activity, it must accurately predict behavior for any object recognition task. To date, we have focused on core object recognition-a remarkable behavior that is accomplished with image viewing durations of \<200 msec. Our work thus far reveals that the neural encoding process is reasonably well explained by a largely feed-forward, highly complex, multistaged nonlinear neural network-the current best neuronal simulation models predict approximately one-half of the relevant neuronal response variance across the highest levels of the ventral stream (areas V4 and IT). Remarkably, however, the decoding process from IT to behavior for all object recognition tasks tested thus far is very accurately predicted by simple direct linear conversion of the inferior temporal neural population state to behavior choice. We have recently examined the behavioral consequences of direct suppression of IT neural activity using pharmacological and optogenetic methods and find them to be well-explained by the same linear decoding model.

}, issn = {0091-7451}, doi = {10.1101/sqb.2014.79.024729}, url = {http://symposium.cshlp.org/content/79/99.full.pdf+html}, author = {Afraz, Arash and Yamins, Daniel L.K. and DiCarlo, James J.} } @article {35, title = {Shape Similarity, Better than Semantic Membership, Accounts for the Structure of Visual Object Representations in a Population of Monkey Inferotemporal Neurons}, journal = {PLoS Computational Biology}, volume = {9}, year = {2013}, month = {08/2013}, pages = {e1003167}, abstract = {

The anterior inferotemporal cortex (IT) is the highest stage along the hierarchy of visual areas that, in primates, processes visual objects. Although several lines of evidence suggest that IT primarily represents visual shape information, some recent studies have argued that neuronal ensembles in IT code the semantic membership of visual objects (i.e., represent conceptual classes such as animate and inanimate objects). In this study, we investigated to what extent semantic, rather than purely visual information, is represented in IT by performing a multivariate analysis of IT responses to a set of visual objects. By relying on a variety of machine-learning approaches (including a cutting-edge clustering algorithm that has been recently developed in the domain of statistical physics), we found that, in most instances, IT representation of visual objects is accounted for by their similarity at the level of shape or, more surprisingly, low-level visual properties. Only in a few cases we observed IT representations of semantic classes that were not explainable by the visual similarity of their members. Overall, these findings reassert the primary function of IT as a conveyor of explicit visual shape information, and reveal that low-level visual properties are represented in IT to a greater extent than previously appreciated. In addition, our work demonstrates how combining a variety of state-of-the-art multivariate approaches, and carefully estimating the contribution of shape similarity to the representation of object categories, can substantially advance our understanding of neuronal coding of visual objects in cortex.

}, doi = {10.1371/journal.pcbi.1003167}, url = {https://dx.plos.org/10.1371/journal.pcbi.1003167}, author = {Baldassi, Carlo and Alemi-Neissi, Alireza and Pagan, Marino and DiCarlo, James J. and Zecchina, Riccardo and Zoccolan, Davide} } @proceedings {48, title = {Towards large-scale, high resolution maps of object selectivity in inferior temporal cortex}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2010}, month = {02/2010}, address = {Salt Lake City, Utah, USA}, abstract = {

Inferior temporal cortex (IT) has been shown to have large-scale (mm to cm) maps of object category selectivity as well as small-scale (sub-millimeter) organization for object features.\  These two scales of spatial organization have yet to be linked because they were measured using different techniques (fMRI, optical imaging, and local electrophysiology), each with their own limitations.\  For example, fMRI has poor spatial resolution while optical imaging has higher resolution at the expense of a narrow field of view of only surface-accessible cortex.\  Given that much of IT lies inside a major cortical sulcus or at the skull base, what is needed is a method that can access the whole of IT with high resolution.\  Microelectrode based mapping has such potential: electrodes can reach almost anywhere in IT (high spatial coverage) and record from single cells (high spatial resolution).\ This potential has not yet been realized because of the difficulty of precisely localizing and co-registering many electrode recordings in vivo.\  Methods such as histological reconstruction of lesion sites or MRI visualization of electrodes are post-hoc and can introduce additional spatial errors.\  Here, we have adopted a microfocal stereo x-ray system for localizing electrodes that can be used at an unlimited number of sites and operates virtually in real-time (Cox et al., J. Neurophys. 2008).\  We have used this system to construct broad scale maps of object category selectivity in IT for comparison to fMRI-based maps.\  We found a weak but significant correspondence between physiology and fMRI maps collected in the same animal, and this correspondence improved substantially when MUA and LFP signals were smoothed (~3-5 mm) to broader scales suggesting the spatial low pass nature of fMRI.\  Transformations other than spatial smoothing such as dividing the LFP into power in different frequency bands did not produce noticeable improvement in map correspondence.

}, doi = {10.3389/conf.fnins.2010.03.00154}, url = {https://www.frontiersin.org/Community/AbstractDetails.aspx?ABS_DOI=10.3389/conf.fnins.2010.03.00154\&eid=770\&sname=computational_and_systems_neur}, author = {Issa, Elias B. and Papanastassiou, A. M. and Andken, BB and DiCarlo, James J.} } @proceedings {47, title = {What is the middle face patch?}, journal = {Society for Neuroscience}, volume = {40}, year = {2010}, month = {2010}, pages = {581.8}, publisher = {SFN }, address = {San Diego, CA, USA}, abstract = {

Functional MRI in the monkey has revealed at least six patches of tissue in the ventral visual cortical stream that prefer images of faces over non-face objects. Neuronal recordings targeted to the largest patch (\“middle face patch\”), hypothesized to be homologous to the human FFA, revealed that nearly all visually driven cells were at least 2x more responsive to faces over non-face images. These results support a view in which this region consists entirely of \“face cells\” involved exclusively in face processing. However, previous work could not distinguish between a single large cluster, several clusters, or a local, graded enrichment of \“face cells\”. More deeply, while \“face cells\” are typically defined as those with a preference for face stimuli on average, it is not clear what model best explains their response properties. A strong form of the \“face cell\” hypothesis is a semantic response model in which each cell responds only to images that an observer would distinguish as a face. An alternative class of models places these cells at an intermediate processing stage along the ventral visual feature hierarchy in which some face selectivity has been achieved, but is the result of neuronal tuning for complex visual features found in both face and some non-face objects. To explore these questions, we used a custom built high resolution x-ray imaging system to map the spiking selectivity at hundreds of sites in and around the fMRI-identified macaque middle face patch using a large class of face and non-face stimuli. Our current data show a single region of face selectivity, approximately 2-3 mm in diameter, in rough correspondence with the fMRI-identified face patch. We confirmed previous results in that ~90\% of neuronal sites in this region are \“face selective\” when evaluated with simple face vs. object response criteria. However, we found that none of these sites maintained a strict semantic designation by responding significantly more to all face images than to any non face image. Moreover, at least ~60\% of sites show a significantly greater response to at least one non-face image, such as images of fruits, cars, and toys, over at least one face image, such as caricatures of human and monkey faces. These results argue against a strong semantic response model of the middle \“face patch.\” Instead, the most parsimonious interpretation of these data is that neurons in the fMRI-identified middle \“face patch\” are located at an intermediate stage of visual processing: although they may be tapped at higher areas to build a single-neuron explicit, semantic level representation of faces, their role may be to represent intermediate visual features that are also found in non-face objects.

}, url = {https://www.abstractsonline.com/Plan/ViewAbstract.aspx?sKey=e08f5ff4-1ba9-4faf-a459-5c9d4be0a1bf\&cKey=36fa0d7d-3e83-4910-be75-57d361ae9e58\&mKey=\%7bE5D5C83F-CE2D-4D71-9DD6-FC7231E090FB\%7d}, author = {Aparicio, Paul L. and Issa, Elias B. and DiCarlo, James J.} } @proceedings {66, title = {A systematic exploration of the relationship of fMRI signals and neuronal activity in the primate temporal lobe}, journal = {Society for Neuroscience}, year = {2009}, month = {11/2009}, publisher = {SFN}, address = {Washington, DC, USA}, author = {Papanastassiou, A. M. and Op de Beeck, H. P. and Andken, BB and DiCarlo, James J.} } @article {64, title = {High-Resolution Three-Dimensional Microelectrode Brain Mapping Using Stereo Microfocal X-ray Imaging}, journal = {Journal of Neurophysiology}, volume = {100}, year = {2008}, month = {01/2008}, pages = {2966 - 2976}, abstract = {

Much of our knowledge of brain function has been gleaned from studies using microelectrodes to characterize the response properties of individual neurons in vivo. However, because it is difficult to accurately determine the location of a microelectrode tip within the brain, it is impossible to systematically map the fine three-dimensional spatial organization of many brain areas, especially in deep structures. Here, we present a practical method based on digital stereo microfocal X-ray imaging that makes it possible to estimate the three-dimensional position of each and every microelectrode recording site in \"real time\" during experimental sessions. We determined the system\&$\#$39;s ex vivo localization accuracy to be better than 50 microm, and we show how we have used this method to coregister hundreds of deep-brain microelectrode recordings in monkeys to a common frame of reference with median error of {\textless}150 microm. We further show how we can coregister those sites with magnetic resonance images {(MRIs),} allowing for comparison with anatomy, and laying the groundwork for more detailed electrophysiology/functional {MRI} comparison. Minimally, this method allows one to marry the single-cell specificity of microelectrode recording with the spatial mapping abilities of imaging techniques; furthermore, it has the potential of yielding fundamentally new kinds of high-resolution maps of brain function.

}, issn = {0022-3077}, doi = {10.1152/jn.90672.2008}, url = {https://www.physiology.org/doi/10.1152/jn.90672.2008}, author = {Cox, David D. and Papanastassiou, Alexander M. and Oreper, Daniel and Andken, Benjamin B. and DiCarlo, James J.} }