@conference {11, title = {Brain-Like Object Recognition with High-Performing Shallow Recurrent ANNs}, booktitle = {Neural Information Processing Systems}, year = {2019}, abstract = {

Deep convolutional artificial neural networks (ANNs) are the leading class of candidate models of the mechanisms of visual processing in the primate ventral stream. While initially inspired by brain anatomy, over the past years, these ANNs have evolved from a simple eight-layer architecture in AlexNet to extremely deep and branching architectures, demonstrating increasingly better object categorization performance, yet bringing into question how brain-like they still are. In particular, typical deep models from the machine learning community are often hard to map onto the brain\&$\#$39;s anatomy due to their vast number of layers and missing biologically-important connections, such as recurrence. Here we demonstrate that better anatomical alignment to the brain and high performance on machine learning as well as neuroscience measures do not have to be in contradiction. We developed CORnet-S, a shallow ANN with four anatomically mapped areas and recurrent connectivity, guided by Brain-Score, a new large-scale composite of neural and behavioral benchmarks for quantifying the functional fidelity of models of the primate ventral visual stream. Despite being significantly shallower than most models, CORnet-S is the top model on Brain-Score and outperforms similarly compact models on ImageNet. Moreover, our extensive analyses of CORnet-S circuitry variants reveal that recurrence is the main predictive factor of both Brain-Score and ImageNet top-1 performance. Finally, we report that the temporal evolution of the CORnet-S \"IT\" neural population resembles the actual monkey IT population dynamics. Taken together, these results establish CORnet-S, a compact, recurrent ANN, as the current best model of the primate ventral visual stream.

}, doi = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns}, url = {https://papers.nips.cc/paper/9441-brain-like-object-recognition-with-high-performing-shallow-recurrent-anns.pdf}, author = {Jonas Kubilius and Martin Schrimpf and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kohitij Kar and Bashivan, Pouya and Jonathan Prescott-Roy and Kailyn Schmidt and Aran Nayebi and Daniel Bear and Daniel L. K. Yamins and James J. DiCarlo} } @article {22, title = {Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?}, journal = {bioRxiv}, year = {2018}, month = {09/2018}, type = {preprint}, abstract = {

The internal representations of early deep artificial neural networks (ANNs) were found to be remarkably similar to the internal neural representations measured experimentally in the primate brain. Here we ask, as deep ANNs have continued to evolve, are they becoming more or less brain-like? ANNs that are most functionally similar to the brain will contain mechanisms that are most like those used by the brain. We therefore developed Brain-Score - a composite of multiple neural and behavioral benchmarks that score any ANN on how similar it is to the brain\&$\#$39;s mechanisms for core object recognition - and we deployed it to evaluate a wide range of state-of-the-art deep ANNs. Using this scoring system, we here report that: (1) DenseNet-169, CORnet-S and ResNet-101 are the most brain-like ANNs. (2) There remains considerable variability in neural and behavioral responses that is not predicted by any ANN, suggesting that no ANN model has yet captured all the relevant mechanisms. (3) Extending prior work, we found that gains in ANN ImageNet performance led to gains on Brain-Score. However, correlation weakened at \>= 70\% top-1 ImageNet performance, suggesting that additional guidance from neuroscience is needed to make further advances in capturing brain mechanisms. (4) We uncovered smaller (i.e. less complex) ANNs that are more brain-like than many of the best-performing ImageNet models, which suggests the opportunity to simplify ANNs to better understand the ventral stream. The scoring system used here is far from complete. However, we propose that evaluating and tracking model-benchmark correspondences through a Brain-Score that is regularly updated with new brain data is an exciting opportunity: experimental benchmarks can be used to guide machine network evolution, and machine networks are mechanistic hypotheses of the brain\&$\#$39;s network and thus drive next experiments. To facilitate both of these, we release Brain-Score.org: a platform that hosts the neural and behavioral benchmarks, where ANNs for visual processing can be submitted to receive a Brain-Score and their rank relative to other models, and where new experimental data can be naturally incorporated.

}, doi = {https://doi.org/10.1101/407007}, url = {https://www.biorxiv.org/content/10.1101/407007v2.full.pdf}, author = {Martin Schrimpf and Kubilius, Jonas and Ha Hong and Najib Majaj and Rajalingham, Rishi and Issa, Elias B. and Kar, Kohitij and Bashivan, Pouya and Jonathan Prescott-Roy and Schmidt, Kailyn and Daniel L. K. Yamins and DiCarlo, James J.} } @article {40, title = {Balanced Increases in Selectivity and Tolerance Produce Constant Sparseness along the Ventral Visual Stream}, journal = {Journal of Neuroscience}, volume = {32}, year = {2012}, month = {07/2012}, pages = {10170 - 10182}, abstract = {

Although popular accounts suggest that neurons along the ventral visual processing stream become increasingly selective for particular objects, this appears at odds with the fact that inferior temporal cortical (IT) neurons are broadly tuned. To explore this apparent contradiction, we compared processing in two ventral stream stages (visual cortical areas V4 and IT) in the rhesus macaque monkey. We confirmed that IT neurons are indeed more selective for conjunctions of visual features than V4 neurons and that this increase in feature conjunction selectivity is accompanied by an increase in tolerance (\"invariance\") to identity-preserving transformations (e.g., shifting, scaling) of those features. We report here that V4 and IT neurons are, on average, tightly matched in their tuning breadth for natural images (\"sparseness\") and that the average V4 or IT neuron will produce a robust firing rate response (\>50\% of its peak observed firing rate) to \∼10\% of all natural images. We also observed that sparseness was positively correlated with conjunction selectivity and negatively correlated with tolerance within both V4 and IT, consistent with selectivity-building and invariance-building computations that offset one another to produce sparseness. Our results imply that the conjunction-selectivity-building and invariance-building computations necessary to support object recognition are implemented in a balanced manner to maintain sparseness at each stage of processing.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.6125-11.2012}, url = {http://www.jneurosci.org/content/32/30/10170.full.pdf+html}, author = {Rust, N. C. and DiCarlo, J. J.} } @article {55, title = {Balanced increases in selectivity and invariance produce constant sparseness across the ventral visual pathway}, journal = {Journal of Vision}, volume = {9}, year = {2009}, month = {Jan-08-2009}, pages = {738 - 738}, abstract = {

While several studies report neurons in inferotemporal cortex (IT) that are highly selective for particular objects or images, other studies report that neurons in IT tend to be broadly tuned. To investigate how selectivity changes across the ventral visual pathway, we compared the responses of neurons in a mid-level visual area (V4) and a high-level visual area (IT). We first assessed the selectivity of neurons in each area by determining how well each population could discriminate between natural images and \“scrambled\” versions of those images that have the same low-level structure but configured randomly. We found that the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled images. These results suggest that IT neurons are in fact more selective than V4 neurons in terms of the image features that drive these cells. As a second estimate of selectivity, we measured the tuning bandwidth of neurons for natural images (\“sparseness\”). Surprisingly, we found that distributions of sparseness values were indistinguishable between V4 and IT. How can the selectivity for natural image features increase while the tuning bandwidth for natural images remains constant? One possibility is that increases in selectivity for particular image features are offset by increases in tolerance for the (e.g.) position and scale of those features. We found that indeed, measures of tolerance were higher in IT than V4. These results confirm that neurons increase both their selectivity for image features and their tolerance to changes in the position and scale of those features as signals propagate through the ventral pathway. Remarkably, the rates of increase of these two parameters appear to be set such that the tuning bandwidth for natural images is maintained across each stage of cortical processing.

}, doi = {10.1167/9.8.738}, url = {http://jov.arvojournals.org/Article.aspx?doi=10.1167/9.8.738}, author = {Rust, N. C. and DiCarlo, J. J.} } @proceedings {91, title = {Is the {\textquotedblleft}binding problem{\textquotedblright} a problem in inferiotemporal cortex?}, journal = {Society for Neuroscience}, year = {2005}, month = {11/2005}, publisher = {SFN}, address = {Washington, DC, USA}, author = {Cox, David D. and DiCarlo, James J.} } @article {90, title = {{\textquoteright}Breaking{\textquoteright} position-invariant object recognition}, journal = {Nature Neuroscience}, volume = {8}, year = {2005}, month = {08/2005}, pages = {1145 - 1147}, abstract = {

While it is often assumed that objects can be recognized irrespective of where they fall on the retina, little is known about the mechanisms underlying this ability. By exposing human subjects to an altered world where some objects systematically changed identity during the transient blindness that accompanies eye movements, we induced predictable object confusions across retinal positions, effectively \&$\#$39;breaking\&$\#$39; position invariance. Thus, position invariance is not a rigid property of vision but is constantly adapting to the statistics of the environment.

}, issn = {1097-6256}, doi = {10.1038/nn1519}, url = {http://www.nature.com/articles/nn1519.pdf}, author = {Cox, David D and Meier, Philip and Oertelt, Nadja and DiCarlo, James J} }