@proceedings {43,
	title = {Comparing-State-of-the-Art Visual Features on Invariant Object Recognition Tasks},
	journal = {IEEE Workshop on Applications of Computer Vision (WACV)},
	year = {2011},
	month = {01/2011},
	pages = {463-470},
	publisher = {IEEE},
	address = {Kona, Hawaii, USA},
	abstract = {<p>Tolerance (\&ldquo;invariance\&rdquo;) to identity-preserving image variation (e.g. variation in position, scale, pose, illumination) is a fundamental problem that any visual object recognition system, biological or engineered, must solve. While standard natural image database benchmarks are useful for guiding progress in computer vision, they can fail to probe the ability of a recognition system to solve the invariance problem. Thus, to understand which computational approaches are making progress on solving the invariance problem, we compared and contrasted a variety of state-of-the-art visual representations using synthetic recognition tasks designed to systematically probe invariance. We successfully re-implemented a variety of state-of-the-art visual representations and confirmed their published performance on a natural image benchmark. We here report that most of these representations perform poorly on invariant recognition, but that one representation shows significant performance gains over two baseline representations. We also show how this approach can more deeply illuminate the strengths and weaknesses of different visual representations and thus guide progress on invariant object recognition.</p>
},
	issn = {978-1-4244-9497-2},
	doi = {10.1109/WACV.2011.5711540},
	url = {https://ieeexplore.ieee.org/document/5711540},
	author = {Pinto, Nicolas and Barhomi, Y and Cox, David D. and DiCarlo, James J.}
}
@proceedings {51,
	title = {Human versus machine: comparing visual object recognition systems on a level playing field},
	journal = {Computation and Systems Neuroscience (COSYNE)},
	year = {2010},
	month = {02/2010},
	address = {Salt Lake City, Utah, USA},
	abstract = {<p>It is received wisdom that biological visual systems easily outmatch current artificial systems at complex visual tasks like object recognition. But have the appropriate comparisons been made? Because artificial systems are improving every day, they may surpass human performance some day. We must understand our progress toward reaching that day, because that success is one of several necessary requirements for \&quot;understanding\&quot; visual object recognition. How large (or small) is the difference in performance between current state-of-the-art object recognition systems and the primate visual system? In practice, the performance comparison of any two object recognition systems requires a focus on the computational crux of the problem and sets of images that engage it. Although it is widely believed that tolerance (\&quot;invariance\&quot;) to identity-preserving image variation (e.g. variation in object position, scale, pose, illumination) is critical, systematic comparisons of state-of-the-art artificial visual representations almost always rely on \&quot;natural\&quot; image databases that can fail to probe the ability of a recognition system to solve the invariance problem [Pinto et al PLoS08, COSYNE08, ECCV08, CVPR09]. Thus, to understand how well current state-of-the-art visual representations perform relative to each other, relative to low-level neuronal representations (e.g. retinal-like and V1-like), and relative to high-level representations (e.g. human performance), we tested all of these representations on a common set of visual object recognition tasks that directly engage the invariance problem. Specifically, we used a synthetic testing approach that allows direct engagement of the invariance problem, as well as knowledge and control of all the key parameters that make object recognition challenging. We successfully re-implemented a variety of state-of-the-art visual representations, and we confirmed the high published performance of all of these state-of-the-art representations on large, complex \&quot;natural\&acirc;\&euro; image benchmarks. Surprisingly, we found that most of these representations were weak on our simple synthetic tests of invariant recognition, and only high-level biologically-inspired representations showed performance gains above the neuroscience \&quot;null\&quot; representation (V1-like). While in aggregate, we found that the performance of these state-of-the-art representations pales in comparison to human performance, humans and computers seem to fail in different and potentially enlightening ways when faced with the problem of invariance. We also show how our synthetic testing approach can more deeply illuminate the strengths and weaknesses of different visual representations and thus guide progress on invariant object recognition.</p>
},
	doi = {10.3389/conf.fnins.2010.03.00283},
	url = {https://www.frontiersin.org/Community/AbstractDetails.aspx?ABS_DOI=10.3389/conf.fnins.2010.03.00283\&eid=770\&sname=Computational_and_Systems_Neuroscience_2010},
	author = {Pinto, Nicolas and Majaj, Najib J and Barhomi, Y and Solomon, E. A. and Cox, David D. and DiCarlo, James J.}
}
