@article {15, title = {Neural dynamics at successive stages of the ventral visual stream are consistent with hierarchical error signals.}, journal = {eLife}, volume = {7}, year = {2018}, month = {11/2018}, abstract = {

Ventral visual stream neural responses are dynamic, even for static image presentations. However, dynamical neural models of visual cortex are lacking as most progress has been made modeling static, time-averaged responses. Here, we studied population neural dynamics during face detection across three cortical processing stages. Remarkably,~30 milliseconds after the initially evoked response, we found that neurons in intermediate level areas decreased their responses to typical configurations of their preferred face parts relative to their response for atypical configurations even while neurons in higher areas achieved and maintained a preference for typical configurations. These hierarchical neural dynamics were inconsistent with standard feedforward circuits. Rather, recurrent models computing prediction errors between stages captured the observed temporal signatures. This model of neural dynamics, which simply augments the standard feedforward model of online vision, suggests that neural responses to static images may encode top-down prediction errors in addition to bottom-up feature estimates.

}, keywords = {Animals, Brain Mapping, Face, Humans, Macaca mulatta, Models, Neurological, Neurons, Pattern Recognition, Photic Stimulation, Reaction Time, Visual, Visual Cortex, Visual Perception}, issn = {2050-084X}, doi = {10.7554/eLife.42870}, url = {https://elifesciences.org/articles/42870https://cdn.elifesciences.org/articles/42870/elife-42870-v2.pdf}, author = {Issa, Elias B and Cadieu, Charles F and DiCarlo, James J} } @article {88, title = {Fast Readout of Object Identity from Macaque Inferior Temporal Cortex}, journal = {Science}, volume = {310}, year = {2005}, month = {04/2005}, pages = {863 - 866}, abstract = {

Understanding the brain computations leading to object recognition requires quantitative characterization of the information represented in inferior temporal (IT) cortex. We used a biologically plausible, classifier-based readout technique to investigate the neural coding of selectivity and invariance at the IT population level. The activity of small neuronal populations (approximately 100 randomly selected cells) over very short time intervals (as small as 12.5 milliseconds) contained unexpectedly accurate and robust information about both object \"identity\" and \"category.\" This information generalized over a range of object positions and scales, even for novel objects. Coarse information about position and scale could also be read out from the same population.

}, keywords = {Action Potentials, Animals, Brain Mapping, Macaca mulatta, Neurons, Psychology, Psychomotor Performance, Recognition, Temporal Lobe, Time Factors, Visual Perception}, issn = {0036-8075}, doi = {10.1126/science.1117593}, url = {https://www.sciencemag.org/lookup/doi/10.1126/science.1117593}, author = {Hung, Chou P. and Kreiman, Gabriel and Poggio, Tomaso and DiCarlo, James J.} } @article {87, title = {Multiple Object Response Normalization in Monkey Inferotemporal Cortex}, journal = {Journal of Neuroscience}, volume = {25}, year = {2005}, month = {07/2005}, pages = {8150 - 8164}, abstract = {

The highest stages of the visual ventral pathway are commonly assumed to provide robust representation of object identity by disregarding confounding factors such as object position, size, illumination, and the presence of other objects (clutter). However, whereas neuronal responses in monkey inferotemporal cortex (IT) can show robust tolerance to position and size changes, previous work shows that responses to preferred objects are usually reduced by the presence of nonpreferred objects. More broadly, we do not yet understand multiple object representation in IT. In this study, we systematically examined IT responses to pairs and triplets of objects in three passively viewing monkeys across a broad range of object effectiveness. We found that, at least under these limited clutter conditions, a large fraction of the response of each IT neuron to multiple objects is reliably predicted as the average of its responses to the constituent objects in isolation. That is, multiple object responses depend primarily on the relative effectiveness of the constituent objects, regardless of object identity. This average effect becomes virtually perfect when populations of IT neurons are pooled. Furthermore, the average effect cannot simply be explained by attentional shifts but behaves as a primarily feedforward response property. Together, our observations are most consistent with mechanistic models in which IT neuronal outputs are normalized by summed synaptic drive into IT or spiking activity within IT and suggest that normalization mechanisms previously revealed at earlier visual areas are operating throughout the ventral visual stream.

}, keywords = {Animals, Brain Mapping, Macaca mulatta, Male, Photic Stimulation, Posture, Psychology, Recognition, Temporal Lobe, Visual Pathways, Visual Perception}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.2058-05.2005}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.2058-05.2005}, author = {Zoccolan, D. and Cox, David D. and DiCarlo, James J.} } @article {89, title = {Using Neuronal Latency to Determine Sensory{\textendash}Motor Processing Pathways in Reaction Time Tasks}, journal = {Journal of Neurophysiology}, volume = {93}, year = {2004}, month = {11/2004}, pages = {2974 - 2986}, abstract = {

We describe a new technique that uses the timing of neuronal and behavioral responses to explore the contributions of individual neurons to specific behaviors. The approach uses both the mean neuronal latency and the trial-by-trial covariance between neuronal latency and behavioral response. Reliable measurements of these values were obtained from single-unit recordings made from anterior inferotemporal (AIT) cortex and the frontal eye fields (FEF) in monkeys while they performed a choice reaction time task. These neurophysiological data show that the responses of AIT neurons and some FEF neurons have little covariance with behavioral response, consistent with a largely \"sensory\" response. The responses of another group of FEF neurons with longer mean latency covary tightly with behavioral response, consistent with a largely \"motor\" response. A very small fraction of FEF neurons had responses consistent with an intermediate position in the sensory-motor pathway. These results suggest that this technique is a valuable tool for exploring the functional organization of neuronal circuits that underlie specific behaviors.

}, keywords = {Action Potentials, Afferent, Animal, Animals, Behavior, Macaca mulatta, Male, Models, Motor Neurons, Neural Pathways, Neurological, Neurons, Photic Stimulation, Psychomotor Performance, Reaction Time, Task Performance and Analysis, Temporal Lobe, Time Factors, Visual Fields}, issn = {0022-3077}, doi = {10.1152/jn.00508.2004}, url = {https://www.physiology.org/doi/10.1152/jn.00508.2004}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @article {99, title = {Anterior Inferotemporal Neurons of Monkeys Engaged in Object Recognition Can be Highly Sensitive to Object Retinal Position}, journal = {Journal of Neurophysiology}, volume = {89}, year = {2003}, month = {01/2003}, pages = {3264 - 3278}, abstract = {

Visual object recognition is computationally difficult because changes in an object\&$\#$39;s position, distance, pose, or setting may cause it to produce a different retinal image on each encounter. To robustly recognize objects, the primate brain must have mechanisms to compensate for these variations. Although these mechanisms are poorly understood, it is thought that they elaborate neuronal representations in the inferotemporal cortex that are sensitive to object form but substantially invariant to other image variations. This study examines this hypothesis for image variation resulting from changes in object position. We studied the effect of small differences (+/-1.5 degrees ) in the retinal position of small (0.6 degrees wide) visual forms on both the behavior of monkeys trained to identify those forms and the responses of 146 anterior IT (AIT) neurons collected during that behavior. Behavioral accuracy and speed were largely unaffected by these small changes in position. Consistent with previous studies, many AIT responses were highly selective for the forms. However, AIT responses showed far greater sensitivity to retinal position than predicted from their reported receptive field (RF) sizes. The median AIT neuron showed a approximately 60\% response decrease between positions within +/-1.5 degrees of the center of gaze, and 52\% of neurons were unresponsive to one or more of these positions. Consistent with previous studies, each neuron\&$\#$39;s rank order of target preferences was largely unaffected across position changes. Although we have not yet determined the conditions necessary to observe this marked position sensitivity in AIT responses, we rule out effects of spatial-frequency content, eye movements, and failures to include the RF center. To reconcile this observation with previous studies, we hypothesize that either AIT position sensitivity strongly depends on object size or that position sensitivity is sharpened by extensive visual experience at fixed retinal positions or by the presence of flanking distractors.

}, keywords = {Action Potentials, Animals, Depth Perception, Electrophysiology, Eye Movements, Form Perception, Macaca mulatta, Male, Neurons, Pattern Recognition, Photic Stimulation, Psychomotor Performance, Retina, Temporal Lobe, Time Factors, Visual, Visual Fields, Visual Perception}, issn = {0022-3077}, doi = {10.1152/jn.00358.2002}, url = {https://www.physiology.org/doi/10.1152/jn.00358.2002}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @article {100, title = {Receptive field structure in cortical area 3b of the alert monkey}, journal = {Behavioural Brain Research}, volume = {135}, year = {2002}, month = {01/2002}, pages = {167 - 178}, abstract = {

More than 350 neurons with fingerpad receptive fields (RFs) were studied in cortical area 3b of three alert monkeys. Random dot patterns, which contain all stimulus patterns with equal probability, were scanned across these RFs at three velocities and eight directions to reveal the RFs\’ spatial and temporal structure. Area 3b RFs are characterized by three components: (1) a single, central excitatory region of short duration, (2) one or more inhibitory regions, also of short duration, that are adjacent to and nearly synchronous with the excitation, and (3) a region of inhibition that overlaps the excitation partially or totally and is temporally delayed with respect to the first two components. As a result of these properties, RF spatial structure depends on scanning direction but is virtually unaffected by changes in scanning velocity. This RF characterization, which is derived solely from responses to scanned random-dot patterns, predicts a neuron\&$\#$39;s responses to random patterns accurately, as expected, but it also predicts orientation sensitivity and preferred orientation measured with a scanned bar. Both orientation sensitivity and the ratio of coincident inhibition (number 2 above) to excitation are stronger in the supra- and infragranular layers than in layer IV.

}, keywords = {Action Potentials, Afferent, Animals, Brain Mapping, Evoked Potentials, Haplorhini, Models, Neurological, Neurons, Orientation, Reproducibility of Results, Skin, Somatosensory, Somatosensory Cortex}, issn = {01664328}, doi = {10.1016/S0166-4328(02)00162-6}, url = {https://linkinghub.elsevier.com/retrieve/pii/S0166432802001626}, author = {DiCarlo, James J and Johnson, Kenneth O} } @article {105, title = {Form representation in monkey inferotemporal cortex is virtually unaltered by free viewing}, journal = {Nature Neuroscience}, volume = {3}, year = {2000}, month = {01/2000}, pages = {814 - 821}, abstract = {

How are objects represented in the brain during natural behavior? Visual object recognition in primates is thought to depend on the inferotemporal cortex {(IT).} In most neurophysiological studies of {IT,} monkeys hold their direction of gaze fixed while isolated visual stimuli are presented (controlled viewing). However, during natural behavior, primates visually explore cluttered environments by changing gaze direction several times each second (free viewing). We examined the effect of free viewing on {IT} neuronal responses in monkeys engaged in a form-recognition task. By making small, real-time stimulus adjustments, we produced nearly identically retinal stimulation during controlled and free viewing. Nearly 90\% of neuronal responses were unaffected by free viewing, and average stimulus selectivity was unchanged. Thus, neuronal representations that likely underlie form recognition are virtually unaltered by free viewing.

}, keywords = {Animals, Conditioning, Fixation, Form Perception, Macaca mulatta, Male, Neurons, Ocular, Pattern Recognition, Photic Stimulation, Psychology, Saccades, Temporal Lobe, Visual, Visual Cortex}, issn = {1097-6256}, doi = {10.1038/77722}, url = {http://www.nature.com/articles/nn0800_814}, author = {DiCarlo, James J. and Maunsell, John H. R.} } @article {103, title = {Spatial and Temporal Structure of Receptive Fields in Primate Somatosensory Area 3b: Effects of Stimulus Scanning Direction and Orientation}, journal = {The Journal of Neuroscience}, volume = {20}, year = {2000}, month = {01/2000}, pages = {495 - 510}, abstract = {

This is the third in a series of studies of the neural representation of tactile spatial form in somatosensory cortical area 3b of the alert monkey. We previously studied the spatial structure of \>350 fingerpad receptive fields (RFs) with random-dot patterns scanned in one direction (DiCarlo et al., 1998) and at varying velocities (DiCarlo and Johnson, 1999). Those studies showed that area 3b RFs have a wide range of spatial structures that are virtually unaffected by changes in scanning velocity. In this study, 62 area 3b neurons were studied with three to eight scanning directions (58 with four or more directions). The data from all three studies are described accurately by an RF model with three components: (1) a single, central excitatory region of short duration, (2) one or more inhibitory regions, also of short duration, that are adjacent to and nearly synchronous with the excitation, and (3) a region of inhibition that overlaps the excitation partially or totally and is temporally delayed with respect to the first two components. The mean correlation between the observed RFs and the RFs predicted by this three-component model was 0.81. The three-component RFs also predicted orientation sensitivity and preferred orientation to a scanned bar accurately. The orientation sensitivity was determined most strongly by the intensity of the coincident RF inhibition in relation to the excitation. Both orientation sensitivity and this ratio were stronger in the supragranular and infragranular layers than in layer IV.

}, keywords = {Action Potentials, Animals, Discrimination Learning, Fingers, Macaca mulatta, Movement, Normal Distribution, Reaction Time, Somatosensory Cortex, Space Perception, Time Factors, Touch}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.20-01-00495.2000}, url = {http://www.jneurosci.org/lookup/doi/10.1523/JNEUROSCI.20-01-00495.2000}, author = {DiCarlo, James J. and Johnson, Kenneth O.} } @article {107, title = {Velocity Invariance of Receptive Field Structure in Somatosensory Cortical Area 3b of the Alert Monkey}, journal = {The Journal of Neuroscience}, volume = {19}, year = {1999}, month = {01/1999}, pages = {401 - 419}, abstract = {

This is the second in a series of studies of the neural representation of tactile spatial form in cortical area 3b of the alert monkey. We previously studied the spatial structure of 330 area 3b neuronal receptive fields (RFs) on the fingerpad with random dot patterns scanned at one velocity (40 mm/sec;\ DiCarlo et al., 1998). Here, we analyze the temporal structure of 84 neuronal RFs by studying their spatial structure at three scanning velocities (20, 40, and 80 mm/sec). As in the previous study, most RFs contained a single, central, excitatory region and one or more surrounding or flanking inhibitory regions. The mean time delay between skin stimulation and its excitatory effect was 15.5 msec. Except for differences in mean rate, each neuron\’s response and the spatial structure of its RF were essentially unaffected by scanning velocity. This is the expected outcome when excitatory and inhibitory effects are brief and synchronous. However, that interpretation is consistent neither with the reported timing of excitation and inhibition in somatosensory cortex nor with the third study in this series, which investigates the effect of scanning direction and shows that one component of inhibition lags behind excitation. We reconcile these observations by showing that overlapping (in-field) inhibition delayed relative to excitation can produce RF spatial structure that is unaffected by changes in scanning velocity. Regardless of the mechanisms, the velocity invariance of area 3b RF structure is consistent with the velocity invariance of tactile spatial perception (e.g., roughness estimation and form recognition).

}, keywords = {Adaptation, Animals, Brain Mapping, Cortical Synchronization, Evoked Potentials, Female, Macaca mulatta, Male, Neural Inhibition, Physiological, Somatosensory Cortex, Visual Fields}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.19-01-00401.1999}, url = {http://www.jneurosci.org/lookup/doi/10.1523/JNEUROSCI.19-01-00401.1999}, author = {DiCarlo, James J. and Johnson, Kenneth O.} } @article {110, title = {Structure of Receptive Fields in Area 3b of Primary Somatosensory Cortex in the Alert Monkey}, journal = {The Journal of Neuroscience}, volume = {18}, year = {1998}, month = {04/1998}, pages = {2626 - 2645}, abstract = {

We investigated the two-dimensional structure of area 3b neuronal receptive fields (RFs) in three alert monkeys. Three hundred thirty neurons with RFs on the distal fingerpads were studied with scanned, random dot stimuli. Each neuron was stimulated continuously for 14 min, yielding 20,000 response data points. Excitatory and inhibitory components of each RF were determined with a modified linear regression algorithm. Analyses assessing goodness-of-fit, repeatability, and generality of the RFs were developed. Two hundred forty-seven neurons yielded highly repeatable RF estimates, and most RFs accounted for a large fraction of the explainable response of each neuron. Although the area 3b RF structures appeared to be continuously distributed, certain structural generalities were apparent. Most RFs (94\%) contained a single, central region of excitation and one or more regions of inhibition located on one, two, three, or all four sides of the excitatory center. The shape, area, and strength of excitatory and inhibitory RF regions ranged widely. Half the RFs contained almost evenly balanced excitation and inhibition. The findings indicate that area 3b neurons act as local spatiotemporal filters that are maximally excited by the presence of particular stimulus features. We believe that form and texture perception are based on high-level representations and that area 3b is an intermediate stage in the processes leading to these representations. Two possibilities are considered: (1) that these high-level representations are basically somatotopic and that area 3b neurons amplify some features and suppress others, or (2) that these representations are highly transformed and that area 3b effects a step in the transformation.

}, keywords = {Afferent, Animals, Data Interpretation, Electrophysiology, Female, Macaca mulatta, Male, Neural Inhibition, Neurons, Reproducibility of Results, Somatosensory Cortex, Statistical, Touch}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.18-07-02626.1998}, url = {http://www.jneurosci.org/lookup/doi/10.1523/JNEUROSCI.18-07-02626.1998}, author = {DiCarlo, James J. and Johnson, Kenneth O. and Hsiao, Steven S.} } @article {116, title = {Marking microelectrode penetrations with fluorescent dyes}, journal = {Journal of Neuroscience Methods}, volume = {64}, year = {1996}, month = {01/1996}, pages = {75 - 81}, abstract = {

Fluorescent dyes were used to mark and identify the tracks left by extracellular microelectrodes in neurophysiological experiments. Forty-two penetrations were made into the postcentral gyrus of 3 Macaque monkeys with electrodes coated with 1 of 5 fluorescent dyes (Dil, DiO, DiI-C5, PyPO, and Fast Blue). The electrodes were driven at rates ranging from 10 to 1000 \μm/min, to a depth of about 4000 \μm, where a small electrolytic lesion was made. Histological sections were viewed under fluorescent optics and the electrode tracks were reconstructed from the dye traces. Fluorescent traces (width 50\–400 \μm) were observed in 41 of 42 penetrations with 24 traces extending to the lesion site. Of the electrodes driven in less than 3 h, those coated with DiI (88) and DiI-C5 (88) left a trace to the lesion site, while 57\% (47) of the DiO, 40\% (25) of the Fast Blue and only 11\% (19) of the PyPO tracks were fully marked.

This method of marking penetrations can be used with any extracellular recording configuration, does not require tissue sections to be processed or stained, does not require electrical lesions, and causes no detectable tissue damage. Because the dyes fluoresce at different wavelengths, closely spaced tracks can be uniquely identified.

}, keywords = {Animals, Brain, Electrophysiology, Fluorescent Dyes, Macaca mulatta, Microelectrodes, Neurosciences}, issn = {01650270}, doi = {10.1016/0165-0270(95)00113-1}, url = {https://linkinghub.elsevier.com/retrieve/pii/0165027095001131}, author = {DiCarlo, James J. and Lane, John W. and Hsiao, Steven S. and Johnson, Kenneth O.} } @article {119, title = {Stimulus configuration, classical conditioning, and hippocampal function.}, journal = {Psychological Review}, volume = {99}, year = {1992}, month = {04/1992}, pages = {268 - 305}, abstract = {

Hippocampal participation in classical conditioning is described in terms of a multilayer network that portrays stimulus configuration. The network (a) describes behavior in real time, (b) incorporates a layer of \"hidden\" units positioned between input and output units, (c) includes inputs that are connected to the output directly as well as indirectly through the hidden-unit layer, and (d) uses a biologically plausible backpropagation procedure to train the hidden-unit layer. Nodes and connections in the neural network are mapped onto regional cerebellar, cortical, and hippocampal circuits, and the effect of lesions of different brain regions is formally studied. Computer simulations of the following classical conditioning paradigms are presented: acquisition of delay and trace conditioning, extinction, acquisition-extinction series of delay conditioning, blocking, over-shadowing, discrimination acquisition, discrimination reversal, feature-positive discrimination, conditioned inhibition, negative patterning, positive patterning, and generalization. The model correctly describes the effect of hippocampal and cortical lesions in many of these paradigms, as well as neural activity in hippocampus and medial septum during classical conditioning. Some of these results might be extended to the description of anterograde amnesia in human patients.

}, keywords = {Animals, Association Learning, Brain Mapping, Cerebellum, Cerebral Cortex, Classical, Computer Simulation, Conditioning, Hippocampus, Humans, Models, Neural Pathways, Neurological}, issn = {0033-295X}, doi = {10.1037/0033-295X.99.2.268}, url = {http://doi.apa.org/getdoi.cfm?doi=10.1037/0033-295X.99.2.268}, author = {Schmajuk, Nestor A. and DiCarlo, James J.} } @article {122, title = {A neural network approach to hippocampal function in classical conditioning.}, journal = {Behavioral Neuroscience}, volume = {105}, year = {1991}, month = {01/1991}, pages = {82 - 110}, abstract = {

Hippocampal participation in classical conditioning in terms of S. Grossberg\&$\#$39;s (1975) attentional theory is described. According to the theory, pairing of a conditioned stimulus/stimuli (CS) with an unconditioned stimulus/stimuli (UCS) causes both an association of the sensory representation of the CS with the UCS (conditioned reinforcement learning) and an association of the sensory representation of the CS with the drive representation of the UCS (incentive motivation learning). Sensory representations compete for a limited-capacity short-term memory (STM). The STM regulation hypothesis, which proposes that the hippocampus controls incentive motivation, self-excitation, and competition among sensory representations thereby regulating the contents of a limited capacity STM, is introduced. Under the STM regulation hypothesis, nodes and connections in Grossberg\&$\#$39;s neural network are mapped onto regional hippocampal-cerebellar circuits. The resulting neural model provides (a) a framework for understanding the dynamics of information processing and storage in the hippocampus and cerebellum during classical conditioning of the rabbit\&$\#$39;s nictitating membrane, (b) principles for understanding the effect of different hippocampal manipulations on classical conditioning, and (c) novel and testable predictions.\ 

}, keywords = {Animals, Cerebellum, Classical, Computer Simulation, Conditioning, Extinction, Eyelid, Hippocampus, Models, Nerve Net, Neurological, Neurons, Psychological, Rabbits, Reaction Time}, issn = {0735-7044}, doi = {10.1037/0735-7044.105.1.82}, url = {http://doi.apa.org/getdoi.cfm?doi=10.1037/0735-7044.105.1.82}, author = {Schmajuk, Nestor A. and DiCarlo, James J.} }