@article {40, title = {Balanced Increases in Selectivity and Tolerance Produce Constant Sparseness along the Ventral Visual Stream}, journal = {Journal of Neuroscience}, volume = {32}, year = {2012}, month = {07/2012}, pages = {10170 - 10182}, abstract = {

Although popular accounts suggest that neurons along the ventral visual processing stream become increasingly selective for particular objects, this appears at odds with the fact that inferior temporal cortical (IT) neurons are broadly tuned. To explore this apparent contradiction, we compared processing in two ventral stream stages (visual cortical areas V4 and IT) in the rhesus macaque monkey. We confirmed that IT neurons are indeed more selective for conjunctions of visual features than V4 neurons and that this increase in feature conjunction selectivity is accompanied by an increase in tolerance (\"invariance\") to identity-preserving transformations (e.g., shifting, scaling) of those features. We report here that V4 and IT neurons are, on average, tightly matched in their tuning breadth for natural images (\"sparseness\") and that the average V4 or IT neuron will produce a robust firing rate response (\>50\% of its peak observed firing rate) to \∼10\% of all natural images. We also observed that sparseness was positively correlated with conjunction selectivity and negatively correlated with tolerance within both V4 and IT, consistent with selectivity-building and invariance-building computations that offset one another to produce sparseness. Our results imply that the conjunction-selectivity-building and invariance-building computations necessary to support object recognition are implemented in a balanced manner to maintain sparseness at each stage of processing.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.6125-11.2012}, url = {http://www.jneurosci.org/content/32/30/10170.full.pdf+html}, author = {Rust, N. C. and DiCarlo, J. J.} } @article {45, title = {Selectivity and Tolerance ("Invariance") Both Increase as Visual Information Propagates from Cortical Area V4 to IT}, journal = {Journal of Neuroscience}, volume = {30}, year = {2010}, month = {09/2010}, pages = {12978 - 12995}, abstract = {

Our ability to recognize objects despite large changes in position, size, and context is achieved through computations that are thought to increase both the shape selectivity and the tolerance (\"invariance\") of the visual representation at successive stages of the ventral pathway [visual cortical areas V1, V2, and V4 and inferior temporal cortex {(IT)].} However, these ideas have proven difficult to test. Here, we consider how well population activity patterns at two stages of the ventral stream {(V4} and {IT)} discriminate between, and generalize across, different images. We found that both V4 and {IT} encode natural images with similar fidelity, whereas the {IT} population is much more sensitive to controlled, statistical scrambling of those images. Scrambling sensitivity was proportional to receptive field {(RF)} size in both V4 and {IT,} suggesting that, on average, the number of visual feature conjunctions implemented by a V4 or {IT} neuron is directly related to its {RF} size. We also found that the {IT} population could better discriminate between objects across changes in position, scale, and context, thus directly demonstrating a {V4-to-IT} gain in tolerance. This tolerance gain could be accounted for by both a decrease in single-unit sensitivity to identity-preserving transformations (e.g., an increase in {RF} size) and an increase in the maintenance of rank-order object selectivity within the {RF.} These results demonstrate that, as visual information travels from V4 to {IT,} the population representation is reformatted to become more selective for feature conjunctions and more tolerant to identity preserving transformations, and they reveal the single-unit response properties that underlie that reformatting.

}, issn = {0270-6474}, doi = {10.1523/JNEUROSCI.0179-10.2010}, url = {http://www.jneurosci.org/cgi/doi/10.1523/JNEUROSCI.0179-10.2010}, author = {Rust, N. C. and DiCarlo, J. J.} } @article {55, title = {Balanced increases in selectivity and invariance produce constant sparseness across the ventral visual pathway}, journal = {Journal of Vision}, volume = {9}, year = {2009}, month = {Jan-08-2009}, pages = {738 - 738}, abstract = {

While several studies report neurons in inferotemporal cortex (IT) that are highly selective for particular objects or images, other studies report that neurons in IT tend to be broadly tuned. To investigate how selectivity changes across the ventral visual pathway, we compared the responses of neurons in a mid-level visual area (V4) and a high-level visual area (IT). We first assessed the selectivity of neurons in each area by determining how well each population could discriminate between natural images and \“scrambled\” versions of those images that have the same low-level structure but configured randomly. We found that the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled images. These results suggest that IT neurons are in fact more selective than V4 neurons in terms of the image features that drive these cells. As a second estimate of selectivity, we measured the tuning bandwidth of neurons for natural images (\“sparseness\”). Surprisingly, we found that distributions of sparseness values were indistinguishable between V4 and IT. How can the selectivity for natural image features increase while the tuning bandwidth for natural images remains constant? One possibility is that increases in selectivity for particular image features are offset by increases in tolerance for the (e.g.) position and scale of those features. We found that indeed, measures of tolerance were higher in IT than V4. These results confirm that neurons increase both their selectivity for image features and their tolerance to changes in the position and scale of those features as signals propagate through the ventral pathway. Remarkably, the rates of increase of these two parameters appear to be set such that the tuning bandwidth for natural images is maintained across each stage of cortical processing.

}, doi = {10.1167/9.8.738}, url = {http://jov.arvojournals.org/Article.aspx?doi=10.1167/9.8.738}, author = {Rust, N. C. and DiCarlo, J. J.} } @proceedings {69, title = {Concurrent increases in selectivity and tolerance produce constant sparseness across the ventral visual stream}, journal = {Computation and Systems Neuroscience (COSYNE)}, year = {2008}, month = {03/2008}, publisher = {COSYNE}, address = {Salt Lake City, Utah, USA}, abstract = {

Neural coding schemes that minimize the number of neurons activated at any one time (or equivalently maximize \“sparseness\”) are thought to be both metabolically and computationally efficient [reviewed by 1]. But does sparseness increase as signals propagate through the cortex? To investigate this question, we compared the response properties of neurons at different stages along the pathway supporting object recognition, the ventral visual stream. Specifically, we recorded the responses of neurons in a mid-level visual area (V4) and a high-level visual area (anterior inferotemporal cortex, IT) to a large set of natural images while monkeys performed an object detection task. We found that the distributions of sparseness values in V4 and IT were indistinguishable and that most neurons in both areas were broadly tuned. Similarly, individual images activated the same, large fraction of neurons in V4 and IT. Thus it appears that a coding principle is conserved at each level of processing; however, in opposition to theories of sparse coding, the tuning we observed in mid- and high-level vision is more consistent with a broadly distributed coding scheme [see also 2].

If sparseness is not changing, what is happening as signals propagate through the visual system? We began investigating this question by measuring tolerance to position and scale transformations. We found that individual neuron tolerance increased from V4 to IT, and this translated to enhanced performance of the IT over the V4 population on a position- and scale-invariant object recognition task. To determine whether the neurons in each area also differ in terms of the image features that elicit a response, we presented natural and \“scrambled\” images that have the same local structure but configured randomly [3]. We found that V4 neurons responded similarly to both image sets whereas IT neurons responded much more robustly to the natural images. Likewise, the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled as compared to the natural images. These results suggest that IT neurons are more selective than V4 neurons in terms of the image features that drive these cells. Moreover, we found that equivalent sparseness values were correlated with higher levels of selectivity and tolerance in IT as compared to V4. Thus, as signals propagate through the visual system, neurons increase their selectivity for particular image features and, at the same time, neurons increase their tolerance for the position and scale of those features; the rates at which these two factors increase are set such that constant sparseness is maintained at each level of visual processing. Consistent with the observation that the structure of cortex is roughly identical regardless of where it sits in the hierarchy, we speculate that conservation of a broadly distributed coding scheme is an optimal use of resources in equipotential cortex.

References

[1] Sparse coding of sensory inputs. BA Olshausen and DJ Field, Curr Opin Neurobiol., 14:481-7, 2004. [2]Responses of neurons in primary and inferior temporal visual cortices to natural scenes. R Baddeley et al., Proc Biol Sci. 264:1775-83, 1997. [3] A parametric texture model based on joint statistics of complex wavelet coefficients. J Portilla and EP Simoncelli, Int J Comp Vis, 40:49-71, 2000

}, url = {http://www.cosyne.org/c/images/8/8e/Cosyne_pf_new.pdf}, author = {Rust, N. C. and DiCarlo, James J.} } @proceedings {67, title = {Increases in selectivity are offset by increases in tolerance ("invariance") to maintain sparseness across the ventral visual pathway}, journal = {Society for Neuroscience}, year = {2008}, month = {11/2008}, pages = {514.8}, publisher = {SFN}, address = {Washington, DC, USA}, abstract = {

Popular accounts of visual processing suggest that neurons become increasingly selective for particular objects and scenes as signals propagate through the ventral visual pathway, but this hypothesis has proven difficult to test. To investigate this issue systematically, we recorded the responses of neurons in a mid-level visual area (V4) and a high-level visual area (IT) under well-controlled conditions (same monkeys, same task, same region of the visual field, same stimuli). We assessed the selectivity of neurons in each area by determining how well each population could discriminate between natural images and \“scrambled\” versions of those images that have the same local structure but configured randomly. We found that the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled as compared to the natural images. These results suggest that IT neurons are more selective than V4 neurons in terms of the image features that drive these cells. As a second estimate of selectivity, we measured the tuning bandwidth of neurons for natural images (commonly called \“sparseness\”). Surprisingly, we found that distributions of sparseness values were indistinguishable between V4 and IT. Similarly, we found that individual images activated the same fraction of neurons in V4 and IT, suggesting that a coding principle is conserved at each level of processing. How can the selectivity for natural image features increase while the tuning bandwidth for natural images remains constant? One possible explanation is that increases in selectivity for particular image features are offset by increases in tolerance for the (e.g.) position and scale of those features. Indeed, when we measured the tolerance of neurons to changes in the position and scale of images, we found that tolerance increases as signals propagate from V4 to IT. Moreover, we found that equivalent sparseness values were correlated with higher levels of selectivity and tolerance in IT as compared to V4. These results confirm that neurons increase both their selectivity for image features and their tolerance to changes in the position and scale of those features as signals propagate through the ventral visual pathway. Remarkably, the rates of increase of these two parameters appear to be set such that an equally distributed coding scheme is maintained at each level of visual processing.

}, url = {https://www.abstractsonline.com/Plan/ViewAbstract.aspx?sKey=fc0d0a2d-b563-4b41-8311-0805f08bde8a\&cKey=8a2c998e-bc76-4d92-96ac-ad5199da59bf\&mKey=\%7bAFEA068D-D012-4520-8E42-10E4D1AF7944\%7d}, author = {Rust, N. C. and James J. DiCarlo} } @proceedings {72, title = {Inferior temporal cortex robustly signals encounters with new objects, but is not an online representation of the visual world}, journal = {Society for Neuroscience}, year = {2008}, month = {11/2008}, pages = {316.6}, publisher = {SFN}, address = {Washington, DC, USA}, abstract = {

Popular accounts of visual processing suggest that neurons become increasingly selective for particular objects and scenes as signals propagate through the ventral visual pathway, but this hypothesis has proven difficult to test. To investigate this issue systematically, we recorded the responses of neurons in a mid-level visual area (V4) and a high-level visual area (IT) under well-controlled conditions (same monkeys, same task, same region of the visual field, same stimuli). We assessed the selectivity of neurons in each area by determining how well each population could discriminate between natural images and \“scrambled\” versions of those images that have the same local structure but configured randomly. We found that the V4 population discriminated between members of the two image sets with similar fidelity whereas discrimination by the IT population was considerably degraded for the scrambled as compared to the natural images. These results suggest that IT neurons are more selective than V4 neurons in terms of the image features that drive these cells. As a second estimate of selectivity, we measured the tuning bandwidth of neurons for natural images (commonly called \“sparseness\”). Surprisingly, we found that distributions of sparseness values were indistinguishable between V4 and IT. Similarly, we found that individual images activated the same fraction of neurons in V4 and IT, suggesting that a coding principle is conserved at each level of processing. How can the selectivity for natural image features increase while the tuning bandwidth for natural images remains constant? One possible explanation is that increases in selectivity for particular image features are offset by increases in tolerance for the (e.g.) position and scale of those features. Indeed, when we measured the tolerance of neurons to changes in the position and scale of images, we found that tolerance increases as signals propagate from V4 to IT. Moreover, we found that equivalent sparseness values were correlated with higher levels of selectivity and tolerance in IT as compared to V4. These results confirm that neurons increase both their selectivity for image features and their tolerance to changes in the position and scale of those features as signals propagate through the ventral visual pathway. Remarkably, the rates of increase of these two parameters appear to be set such that an equally distributed coding scheme is maintained at each level of visual processing.

}, url = {https://www.abstractsonline.com/Plan/ViewAbstract.aspx?sKey=ee83e7f7-5aea-4ec8-a948-436658d20e37\&cKey=fc64f0af-c81e-4b0e-b809-796349279531\&mKey=\%7bAFEA068D-D012-4520-8E42-10E4D1AF7944\%7d}, author = {Rust, N. C. and DiCarlo, James J.} }