@conference {146, title = {Wiring Up Vision: Minimizing Supervised Synaptic Updates Needed to Produce a Primate Ventral Stream}, booktitle = {International Conference on Learning Representations 2022 Spotlight}, year = {2022}, month = {April 25, 2022}, type = {preprint}, abstract = {

After training on large datasets, certain deep neural networks are surprisingly good models of the neural mechanisms of adult primate visual object recognition. Nevertheless, these models are considered poor models of the development of the visual system because they posit millions of sequential, precisely coordinated synaptic updates, each based on a labeled image. While ongoing research is pursuing the use of unsupervised proxies for labels, we here explore a complementary strategy of reducing the required number of supervised synaptic updates to produce an adult-like ventral visual stream (as judged by the match to V1, V2, V4, IT, and behavior). Such models might require less precise machinery and energy expenditure to coordinate these updates and would thus move us closer to viable neuroscientific hypotheses about how the visual system wires itself up. Relative to standard model training on labeled images in ImageNet, we here demonstrate that the total number of supervised weight updates can be substantially reduced using three complementary strategies: First, we find that only 2\% of supervised updates (epochs and images) are needed to achieve \∼80\% of a fully trained model\’s match to adult ventral stream. Specifically, training benefits predictions of higher visual cortex the most whereas predictions of earlier areas improve only marginally over the course of training. Second, by improving the random distribution of synaptic connectivity, we find that 54\% of the brain match can already be achieved \“at birth\” (i.e. no training at all). Third, we find that, by training only \∼5\% of model synapses, we can still achieve nearly 80\% of the match to the ventral stream. This approach further improves on ImageNet performance over previous attempts in computer vision of minimizing trained components without substantially increasing the number of trained parameters. These results reflect first steps in modeling not just primate adult visual processing during inference, but also how the ventral visual stream might be \“wired up\” by evolution (a model\’s \“birth\” state) and by developmental learning (a model\’s updates based on visual experience).

}, keywords = {biologically plausible learning, computational neuroscience, convolutional neural networks, primate visual ventral stream}, doi = {10.1101/2020.06.08.140111}, url = {https://openreview.net/pdf?id=g1SzIRLQXMM}, author = {Geiger, Franziska and Schrimpf, Martin and Marques, Tiago and DiCarlo, James J} } @proceedings {47, title = {What is the middle face patch?}, journal = {Society for Neuroscience}, volume = {40}, year = {2010}, month = {2010}, pages = {581.8}, publisher = {SFN }, address = {San Diego, CA, USA}, abstract = {

Functional MRI in the monkey has revealed at least six patches of tissue in the ventral visual cortical stream that prefer images of faces over non-face objects. Neuronal recordings targeted to the largest patch (\“middle face patch\”), hypothesized to be homologous to the human FFA, revealed that nearly all visually driven cells were at least 2x more responsive to faces over non-face images. These results support a view in which this region consists entirely of \“face cells\” involved exclusively in face processing. However, previous work could not distinguish between a single large cluster, several clusters, or a local, graded enrichment of \“face cells\”. More deeply, while \“face cells\” are typically defined as those with a preference for face stimuli on average, it is not clear what model best explains their response properties. A strong form of the \“face cell\” hypothesis is a semantic response model in which each cell responds only to images that an observer would distinguish as a face. An alternative class of models places these cells at an intermediate processing stage along the ventral visual feature hierarchy in which some face selectivity has been achieved, but is the result of neuronal tuning for complex visual features found in both face and some non-face objects. To explore these questions, we used a custom built high resolution x-ray imaging system to map the spiking selectivity at hundreds of sites in and around the fMRI-identified macaque middle face patch using a large class of face and non-face stimuli. Our current data show a single region of face selectivity, approximately 2-3 mm in diameter, in rough correspondence with the fMRI-identified face patch. We confirmed previous results in that ~90\% of neuronal sites in this region are \“face selective\” when evaluated with simple face vs. object response criteria. However, we found that none of these sites maintained a strict semantic designation by responding significantly more to all face images than to any non face image. Moreover, at least ~60\% of sites show a significantly greater response to at least one non-face image, such as images of fruits, cars, and toys, over at least one face image, such as caricatures of human and monkey faces. These results argue against a strong semantic response model of the middle \“face patch.\” Instead, the most parsimonious interpretation of these data is that neurons in the fMRI-identified middle \“face patch\” are located at an intermediate stage of visual processing: although they may be tapped at higher areas to build a single-neuron explicit, semantic level representation of faces, their role may be to represent intermediate visual features that are also found in non-face objects.

}, url = {https://www.abstractsonline.com/Plan/ViewAbstract.aspx?sKey=e08f5ff4-1ba9-4faf-a459-5c9d4be0a1bf\&cKey=36fa0d7d-3e83-4910-be75-57d361ae9e58\&mKey=\%7bE5D5C83F-CE2D-4D71-9DD6-FC7231E090FB\%7d}, author = {Aparicio, Paul L. and Issa, Elias B. and DiCarlo, James J.} } @article {53, title = {What Response Properties Do Individual Neurons Need to Underlie Position and Clutter {\textquotedblleft}Invariant{\textquotedblright} Object Recognition?}, journal = {Journal of Neurophysiology}, volume = {102}, year = {2009}, month = {01/2009}, pages = {360 - 376}, abstract = {

Primates can easily identify visual objects over large changes in retinal position\–a property commonly referred to as position \"invariance.\" This ability is widely assumed to depend on neurons in inferior temporal cortex {(IT)} that can respond selectively to isolated visual objects over similarly large ranges of retinal position. However, in the real world, objects rarely appear in isolation, and the interplay between position invariance and the representation of multiple objects (i.e., clutter) remains unresolved. At the heart of this issue is the intuition that the representations of nearby objects can interfere with one another and that the large receptive fields needed for position invariance can exacerbate this problem by increasing the range over which interference acts. Indeed, most {IT} neurons\&$\#$39; responses are strongly affected by the presence of clutter. While external mechanisms (such as attention) are often invoked as a way out of the problem, we show (using recorded neuronal data and simulations) that the intrinsic properties of {IT} population responses, by themselves, can support object recognition in the face of limited clutter. Furthermore, we carried out extensive simulations of hypothetical neuronal populations to identify the essential individual-neuron ingredients of a good population representation. These simulations show that the crucial neuronal property to support recognition in clutter is not preservation of response magnitude, but preservation of each neuron\&$\#$39;s rank-order object preference under identity-preserving image transformations (e.g., clutter). Because {IT} neuronal responses often exhibit that response property, while neurons in earlier visual areas (e.g., V1) do not, we suggest that preserving the rank-order object preference regardless of clutter, rather than the response magnitude, more precisely describes the goal of individual neurons at the top of the ventral visual stream.

}, issn = {0022-3077}, doi = {10.1152/jn.90745.2008}, url = {https://www.physiology.org/doi/10.1152/jn.90745.2008}, author = {Li, Nuo and Cox, David D. and Zoccolan, Davide and DiCarlo, James J.} } @proceedings {77, title = {Why is real-world object recognition hard?: Establishing honest benchmarks and baselines for object recognition}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2008}, month = {03/2008}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, author = {Pinto, Nicolas and Cox, David D. and Corda, B and Doukhan, David and DiCarlo, James J.} } @article {65, title = {Why is Real-World Visual Object Recognition Hard?}, journal = {PLoS Computational Biology}, volume = {4}, year = {2008}, month = {01/2008}, pages = {e27}, abstract = {

Progress in understanding the brain mechanisms underlying vision requires the construction of computational models that not only emulate the brain\&$\#$39;s anatomy and physiology, but ultimately match its performance on visual tasks. In recent years, \"natural\" images have become popular in the study of vision and have been used to show apparently impressive progress in building such models. Here, we challenge the use of uncontrolled \"natural\" images in guiding that progress. In particular, we show that a simple V1-like model\–a neuroscientist\&$\#$39;s \"null\" model, which should perform poorly at real-world visual object recognition tasks\–outperforms state-of-the-art object recognition systems (biologically inspired and otherwise) on a standard, ostensibly natural image recognition test. As a counterpoint, we designed a \"simpler\" recognition test to better span the real-world variation in object pose, position, and scale, and we show that this test correctly exposes the inadequacy of the V1-like model. Taken together, these results demonstrate that tests based on uncontrolled natural images can be seriously misleading, potentially guiding progress in the wrong direction. Instead, we reexamine what it means for images to be natural and argue for a renewed focus on the core problem of object recognition\–real-world image variation.

}, doi = {10.1371/journal.pcbi.0040027}, url = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.0040027}, author = {Pinto, Nicolas and Cox, David D and DiCarlo, James J}, editor = {Friston, Karl J} }