@article {175, title = {Robustified ANNs Reveal Wormholes Between Human Category Percepts}, journal = {arXiv}, year = {2023}, month = {08/2023}, type = {preprint}, abstract = {

The visual object category reports of artificial neural networks (ANNs) are notoriously sensitive to tiny, adversarial image perturbations. Because human category reports (aka human percepts) are thought to be insensitive to those same small-norm perturbations -- and locally stable in general -- this argues that ANNs are incomplete scientific models of human visual perception. Consistent with this, we show that when small-norm image perturbations are generated by standard ANN models, human object category percepts are indeed highly stable. However, in this very same \"human-presumed-stable\" regime, we find that robustified ANNs reliably discover low-norm image perturbations that strongly disrupt human percepts. These previously undetectable human perceptual disruptions are massive in amplitude, approaching the same level of sensitivity seen in robustified ANNs. Further, we show that robustified ANNs support precise perceptual state interventions: they guide the construction of low-norm image perturbations that strongly alter human category percepts toward specific prescribed percepts. These observations suggest that for arbitrary starting points in image space, there exists a set of nearby \"wormholes\", each leading the subject from their current category perceptual state into a semantically very different state. Moreover, contemporary ANN models of biological visual processing are now accurate enough to consistently guide us to those portals.

}, doi = { https://doi.org/10.48550/arXiv.2308.06887 Focus to learn more}, url = {https://arxiv.org/pdf/2308.06887.pdf}, author = {Gaziv, Guy and Lee, Michael J and DiCarlo, James J} } @conference {179, title = {Strong and Precise Modulation of Human Percepts via Robustified ANNs}, booktitle = {Neural Information Processing Systems}, year = {2023}, address = {New Orleans, Louisiana}, abstract = {

The visual object category reports of artificial neural networks (ANNs) are notoriously sensitive to tiny, adversarial image perturbations. Because human category reports (aka human percepts) are thought to be insensitive to those same small-norm perturbations \– and locally stable in general \– this argues that ANNs are incomplete scientific models of human visual perception. Consistent with this, we show that when small-norm image perturbations are generated by standard ANN models, human object category percepts are indeed highly stable. However, in this very same \“human-presumed-stable\” regime, we find that robustified ANNs reliably discover low-norm image perturbations that strongly disrupt human percepts. These previously undetectable human perceptual disruptions are massive in amplitude, approaching the same level of sensitivity seen in robustified ANNs. Further, we show that robustified ANNs support precise perceptual state interventions: they guide the construction of low-norm image perturbations that strongly alter human category percepts toward specific prescribed percepts. In sum, these contemporary models of biological visual processing are now accurate enough to guide strong and precise interventions on human perception.

}, url = {https://openreview.net/pdf?id=5GmTI4LNqX}, author = {Gaziv, Guy and Lee, Michael J and DiCarlo, James J} } @article {170, title = {Adversarially trained neural representations may already be as robust as corresponding biological neural representations}, journal = {arXiv}, year = {2022}, month = {06/19/2022}, type = {preprint}, abstract = {

Visual systems of primates are the gold standard of robust perception. There is thus a general belief that mimicking the neural representations that underlie those systems will yield artificial visual systems that are adversarially robust. In this work, we develop a method for performing adversarial visual attacks directly on primate brain activity. We then leverage this method to demonstrate that the above-mentioned belief might not be well founded. Specifically, we report that the biological neurons that make up visual systems of primates exhibit susceptibility to adversarial perturbations that is comparable in magnitude to existing (robustly trained) artificial neural networks.

}, doi = {https://doi.org/10.48550/arXiv.2206.11228}, url = {https://arxiv.org/abs/2206.11228v1}, author = {Guo, Chong and Lee, Michael J and Leclerc, Guillaume and Dapello, Joel and Rao, Yug and Madry, Aleksander and DiCarlo, James J} } @article {168, title = {Comparing novel object learning in humans, models, and monkeys}, journal = {Journal of Vision}, volume = {19}, year = {2019}, month = {Jun-09-2019}, pages = {114b}, abstract = {

Humans readily learn to identify novel objects, and it has been hypothesized that plasticity in visual cortex supports this behavior. Contributing to this view are reports of experience-driven changes in the properties of neurons at many levels of visual cortex, from V1 to inferotemporal cortex (IT). Here, we ask if object learning might instead be explained by a simple model in which a static set of IT-like visual features is followed by a perceptron learner. Specifically, we measured human (268 subjects; 170,000+ trials) and nonhuman primate (NHP; 2 subjects, 300,000+ trials) behavior across a battery of 29 visuomotor association tasks that each required the subject to learn to discriminate between a pair of synthetically-generated, never-before-seen 3D objects (58 distinct objects). Objects were rendered at varying scales, positions, and rotations; superimposed on naturalistic backgrounds; and presented for 200 msec. We then approximated the visual system\’s IT response to each image using models of ventral stream processing (i.e. specific deep neural networks trained on ImageNet categorization), and we applied a reward-based, perceptron learner to the static set of features produced at the penultimate layer of each model. We report that our model is sufficient to explain both human and NHP rates of learning on these tasks. Additionally, we show humans, NHPs, and this model share the same pattern of performance over objects, but that NHPs reach criterion performance ~10\× as slowly as humans (human t = 139, NHP t = 1149), suggesting humans have similar but more rapid learning mechanisms than their NHP cousins in this domain. Taken together, these results suggest the possibility that object learning is mediated by plasticity in a small population of \“readout\” neurons that learn and execute weighted sums of activity across an upstream sensory population representation (IT) that is largely stable.

}, issn = {1534-7362}, doi = {10.1167/19.10.114b}, url = {https://jov.arvojournals.org/article.aspx?articleid=2750359}, author = {Lee, Michael J and DiCarlo, James J} }