@conference {11, title = {Brain-Like Object Recognition with High-Performing Shallow Recurrent ANNs}, booktitle = {Neural Information Processing Systems}, year = {2019}, abstract = {

Deep convolutional artificial neural networks (ANNs) are the leading class of candidate models of the mechanisms of visual processing in the primate ventral stream. While initially inspired by brain anatomy, over the past years, these ANNs have evolved from a simple eight-layer architecture in AlexNet to extremely deep and branching architectures, demonstrating increasingly better object categorization performance, yet bringing into question how brain-like they still are. In particular, typical deep models from the machine learning community are often hard to map onto the brain\&$\#$39;s anatomy due to their vast number of layers and missing biologically-important connections, such as recurrence. Here we demonstrate that better anatomical alignment to the brain and high performance on machine learning as well as neuroscience measures do not have to be in contradiction. We developed CORnet-S, a shallow ANN with four anatomically mapped areas and recurrent connectivity, guided by Brain-Score, a new large-scale composite of neural and behavioral benchmarks for quantifying the functional fidelity of models of the primate ventral visual stream. Despite being significantly shallower than most models, CORnet-S is the top model on Brain-Score and outperforms similarly compact models on ImageNet. Moreover, our extensive analyses of CORnet-S circuitry variants reveal that recurrence is the main predictive factor of both Brain-Score and ImageNet top-1 performance. Finally, we report that the temporal evolution of the CORnet-S \"IT\" neural population resembles the actual monkey IT population dynamics. Taken together, these results establish CORnet-S, a compact, recurrent ANN, as the current best model of the primate ventral visual stream.

}, doi = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns}, url = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns.pdf}, author = {Jonas Kubilius and Martin Schrimpf and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kohitij Kar and Bashivan, Pouya and Jonathan Prescott-Roy and Kailyn Schmidt and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and James J. DiCarlo} } @article {4, title = {Evidence that recurrent circuits are critical to the ventral stream{\textquoteright}s execution of core object recognition behavior}, journal = {Nature Neuroscience}, volume = {22}, year = {2019}, month = {01/2019}, pages = {974 - 983}, issn = {1097-6256}, doi = {10.1038/s41593-019-0392-5}, url = {http://www.nature.com/articles/s41593-019-0392-5}, author = {Kar, Kohitij and Kubilius, Jonas and Schmidt, Kailyn and Issa, Elias B. and DiCarlo, James J.} } @article {22, title = {Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?}, journal = {bioRxiv}, year = {2018}, month = {09/2018}, type = {preprint}, abstract = {

The internal representations of early deep artificial neural networks (ANNs) were found to be remarkably similar to the internal neural representations measured experimentally in the primate brain. Here we ask, as deep ANNs have continued to evolve, are they becoming more or less brain-like? ANNs that are most functionally similar to the brain will contain mechanisms that are most like those used by the brain. We therefore developed Brain-Score - a composite of multiple neural and behavioral benchmarks that score any ANN on how similar it is to the brain\&$\#$39;s mechanisms for core object recognition - and we deployed it to evaluate a wide range of state-of-the-art deep ANNs. Using this scoring system, we here report that: (1) DenseNet-169, CORnet-S and ResNet-101 are the most brain-like ANNs. (2) There remains considerable variability in neural and behavioral responses that is not predicted by any ANN, suggesting that no ANN model has yet captured all the relevant mechanisms. (3) Extending prior work, we found that gains in ANN ImageNet performance led to gains on Brain-Score. However, correlation weakened at \>= 70\% top-1 ImageNet performance, suggesting that additional guidance from neuroscience is needed to make further advances in capturing brain mechanisms. (4) We uncovered smaller (i.e. less complex) ANNs that are more brain-like than many of the best-performing ImageNet models, which suggests the opportunity to simplify ANNs to better understand the ventral stream. The scoring system used here is far from complete. However, we propose that evaluating and tracking model-benchmark correspondences through a Brain-Score that is regularly updated with new brain data is an exciting opportunity: experimental benchmarks can be used to guide machine network evolution, and machine networks are mechanistic hypotheses of the brain\&$\#$39;s network and thus drive next experiments. To facilitate both of these, we release Brain-Score.org: a platform that hosts the neural and behavioral benchmarks, where ANNs for visual processing can be submitted to receive a Brain-Score and their rank relative to other models, and where new experimental data can be naturally incorporated.

}, doi = {https://doi.org/10.1101/407007}, url = {https://www.biorxiv.org/content/10.1101/407007v2.full.pdf}, author = {Martin Schrimpf and Kubilius, Jonas and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kar, Kohitij and Bashivan, Pouya and Jonathan Prescott-Roy and Schmidt, Kailyn and Daniel L. K. Yamins and DiCarlo, James J.} } @article {13, title = {Large-Scale, High-Resolution Comparison of the Core Visual Object Recognition Behavior of Humans, Monkeys, and State-of-the-Art Deep Artificial Neural Networks}, journal = {The Journal of Neuroscience}, volume = {38}, year = {2018}, month = {03/2019}, pages = {7255 - 7269}, abstract = {

Primates-including humans-can typically recognize objects in visual images at a glance even in spite of naturally occurring identity-preserving image transformations (e.g. changes in viewpoint). A primary neuroscience goal is to uncover neuron-level mechanistic models that quantitatively explain this behavior by predicting primate performance for each and every image. Here, we applied this stringent behavioral prediction test to the leading mechanistic models of primate vision (specifically, deep, convolutional, artificial neural networks; ANNs) by directly comparing their behavioral signatures against those of humans and rhesus macaque monkeys. Using high-throughput data collection systems for human and monkey psychophysics, we collected over one million behavioral trials from 1472 anonymous humans and five male macaque monkeys for 2400 images over 276 binary object discrimination tasks. Consistent with previous work, we observed that state-of-the-art deep, feed-forward convolutional ANNs trained for visual categorization (termed DCNN models) accurately predicted primate patterns of object-level confusion. However, when we examined behavioral performance for individual images within each object discrimination task, we found that all tested DCNN models were significantly non-predictive of primate performance, and that this prediction failure was not accounted for by simple image attributes, nor rescued by simple model modifications. These results show that current DCNN models cannot account for the image-level behavioral patterns of primates, and that new ANN models are needed to more precisely capture the neural mechanisms underlying primate object vision. To this end, large-scale, high-resolution primate behavioral benchmarks-such as those obtained here-could serve as direct guides for discovering such models.Recently, specific feed-forward deep convolutional artificial neural networks (ANNs) models have dramatically advanced our quantitative understanding of the neural mechanisms underlying primate core object recognition. In this work, we tested the limits of those ANNs by systematically comparing the behavioral responses of these models with the behavioral responses of humans and monkeys, at the resolution of individual images. Using these high-resolution metrics, we found that all tested ANN models significantly diverged from primate behavior. Going forward, these high-resolution, large-scale primate behavioral benchmarks could serve as direct guides for discovering better ANN models of the primate visual system.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0388-18.2018}, url = {http://www.jneurosci.org/content/38/33/7255}, author = {Rajalingham, Rishi and Issa, Elias B. and Bashivan, Pouya and Kar, Kohitij and Schmidt, Kailyn and DiCarlo, James J.} } @article {24, title = {Neurophysiological Organization of the Middle Face Patch in Macaque Inferior Temporal Cortex}, journal = {The Journal of Neuroscience}, volume = {36}, year = {2016}, month = {02/2017}, pages = {12729 - 12745}, abstract = {

While early cortical visual areas contain fine scale spatial organization of neuronal properties, such as orientation preference, the spatial organization of higher-level visual areas is less well understood. The fMRI demonstration of face-preferring regions in human ventral cortex and monkey inferior temporal cortex (\“face patches\”) raises the question of how neural selectivity for faces is organized. Here, we targeted hundreds of spatially registered neural recordings to the largest fMRI-identified face-preferring region in monkeys, the middle face patch (MFP), and show that the MFP contains a graded enrichment of face-preferring neurons. At its center, as much as 93\% of the sites we sampled responded twice as strongly to faces than to nonface objects. We estimate the maximum neurophysiological size of the MFP to be \∼6 mm in diameter, consistent with its previously reported size under fMRI. Importantly, face selectivity in the MFP varied strongly even between neighboring sites. Additionally, extremely face-selective sites were \∼40 times more likely to be present inside the MFP than outside. These results provide the first direct quantification of the size and neural composition of the MFP by showing that the cortical tissue localized to the fMRI defined region consists of a very high fraction of face-preferring sites near its center, and a monotonic decrease in that fraction along any radial spatial axis.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0237-16.2016}, url = {https://www.jneurosci.org/content/jneuro/36/50/12729.full.pdf}, author = {Aparicio, Paul L. and Issa, Elias B. and DiCarlo, James J.} } @proceedings {48, title = {Towards large-scale, high resolution maps of object selectivity in inferior temporal cortex}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2010}, month = {02/2010}, address = {Salt Lake City, Utah, USA}, abstract = {

Inferior temporal cortex (IT) has been shown to have large-scale (mm to cm) maps of object category selectivity as well as small-scale (sub-millimeter) organization for object features.\  These two scales of spatial organization have yet to be linked because they were measured using different techniques (fMRI, optical imaging, and local electrophysiology), each with their own limitations.\  For example, fMRI has poor spatial resolution while optical imaging has higher resolution at the expense of a narrow field of view of only surface-accessible cortex.\  Given that much of IT lies inside a major cortical sulcus or at the skull base, what is needed is a method that can access the whole of IT with high resolution.\  Microelectrode based mapping has such potential: electrodes can reach almost anywhere in IT (high spatial coverage) and record from single cells (high spatial resolution).\ This potential has not yet been realized because of the difficulty of precisely localizing and co-registering many electrode recordings in vivo.\  Methods such as histological reconstruction of lesion sites or MRI visualization of electrodes are post-hoc and can introduce additional spatial errors.\  Here, we have adopted a microfocal stereo x-ray system for localizing electrodes that can be used at an unlimited number of sites and operates virtually in real-time (Cox et al., J. Neurophys. 2008).\  We have used this system to construct broad scale maps of object category selectivity in IT for comparison to fMRI-based maps.\  We found a weak but significant correspondence between physiology and fMRI maps collected in the same animal, and this correspondence improved substantially when MUA and LFP signals were smoothed (~3-5 mm) to broader scales suggesting the spatial low pass nature of fMRI.\  Transformations other than spatial smoothing such as dividing the LFP into power in different frequency bands did not produce noticeable improvement in map correspondence.

}, doi = {10.3389/conf.fnins.2010.03.00154}, url = {https://www.frontiersin.org/Community/AbstractDetails.aspx?ABS_DOI=10.3389/conf.fnins.2010.03.00154\&eid=770\&sname=computational_and_systems_neur}, author = {Issa, Elias B. and Papanastassiou, A. M. and Andken, BB and DiCarlo, James J.} } @proceedings {47, title = {What is the middle face patch?}, journal = {Society for Neuroscience}, volume = {40}, year = {2010}, month = {2010}, pages = {581.8}, publisher = {SFN }, address = {San Diego, CA, USA}, abstract = {

Functional MRI in the monkey has revealed at least six patches of tissue in the ventral visual cortical stream that prefer images of faces over non-face objects. Neuronal recordings targeted to the largest patch (\“middle face patch\”), hypothesized to be homologous to the human FFA, revealed that nearly all visually driven cells were at least 2x more responsive to faces over non-face images. These results support a view in which this region consists entirely of \“face cells\” involved exclusively in face processing. However, previous work could not distinguish between a single large cluster, several clusters, or a local, graded enrichment of \“face cells\”. More deeply, while \“face cells\” are typically defined as those with a preference for face stimuli on average, it is not clear what model best explains their response properties. A strong form of the \“face cell\” hypothesis is a semantic response model in which each cell responds only to images that an observer would distinguish as a face. An alternative class of models places these cells at an intermediate processing stage along the ventral visual feature hierarchy in which some face selectivity has been achieved, but is the result of neuronal tuning for complex visual features found in both face and some non-face objects. To explore these questions, we used a custom built high resolution x-ray imaging system to map the spiking selectivity at hundreds of sites in and around the fMRI-identified macaque middle face patch using a large class of face and non-face stimuli. Our current data show a single region of face selectivity, approximately 2-3 mm in diameter, in rough correspondence with the fMRI-identified face patch. We confirmed previous results in that ~90\% of neuronal sites in this region are \“face selective\” when evaluated with simple face vs. object response criteria. However, we found that none of these sites maintained a strict semantic designation by responding significantly more to all face images than to any non face image. Moreover, at least ~60\% of sites show a significantly greater response to at least one non-face image, such as images of fruits, cars, and toys, over at least one face image, such as caricatures of human and monkey faces. These results argue against a strong semantic response model of the middle \“face patch.\” Instead, the most parsimonious interpretation of these data is that neurons in the fMRI-identified middle \“face patch\” are located at an intermediate stage of visual processing: although they may be tapped at higher areas to build a single-neuron explicit, semantic level representation of faces, their role may be to represent intermediate visual features that are also found in non-face objects.

}, url = {https://www.abstractsonline.com/Plan/ViewAbstract.aspx?sKey=e08f5ff4-1ba9-4faf-a459-5c9d4be0a1bf\&cKey=36fa0d7d-3e83-4910-be75-57d361ae9e58\&mKey=\%7bE5D5C83F-CE2D-4D71-9DD6-FC7231E090FB\%7d}, author = {Aparicio, Paul L. and Issa, Elias B. and DiCarlo, James J.} }