{"id":"https://openalex.org/W2611083268","doi":"https://doi.org/10.1145/3025453.3025930","title":"Scalable Annotation of Fine-Grained Categories Without Experts","display_name":"Scalable Annotation of Fine-Grained Categories Without Experts","publication_year":2017,"publication_date":"2017-05-02","ids":{"openalex":"https://openalex.org/W2611083268","doi":"https://doi.org/10.1145/3025453.3025930","mag":"2611083268"},"language":"en","primary_location":{"id":"doi:10.1145/3025453.3025930","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3025453.3025930","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3025930&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=3025930&type=pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089275971","display_name":"Timnit Gebru","orcid":"https://orcid.org/0009-0007-4814-1944"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Timnit Gebru","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060909587","display_name":"Jonathan Krause","orcid":"https://orcid.org/0000-0002-1952-3480"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jonathan Krause","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101542158","display_name":"Jia Deng","orcid":"https://orcid.org/0000-0001-9594-4554"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan\u2013Ann Arbor","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jia Deng","raw_affiliation_strings":["University of Michigan, Ann Arbor, MI, USA"],"affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor, MI, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100450462","display_name":"Li Fei-Fei","orcid":"https://orcid.org/0000-0002-7481-0810"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li Fei-Fei","raw_affiliation_strings":["Stanford University, Stanford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5089275971"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":4.7014,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":{"value":0.94765451,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1877","last_page":"1881"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8858911991119385},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.88197922706604},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.755948543548584},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6952933669090271},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6769763827323914},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.5933067202568054},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.4774523377418518},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4697969853878021},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.4675924479961395},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.4175562858581543},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35051366686820984},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.325181245803833},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.26806724071502686},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.13617059588432312},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.087473064661026},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.07403653860092163}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8858911991119385},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.88197922706604},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.755948543548584},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6952933669090271},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6769763827323914},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.5933067202568054},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.4774523377418518},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4697969853878021},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.4675924479961395},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.4175562858581543},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35051366686820984},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.325181245803833},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.26806724071502686},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.13617059588432312},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.087473064661026},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.07403653860092163},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3025453.3025930","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3025453.3025930","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3025930&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3025453.3025930","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3025453.3025930","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3025930&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1559388177","display_name":null,"funder_award_id":"11154","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7194250716","display_name":"III: Small: Collaborative Research: Using Large-Scale Image Data for Online Social Media Analysis","funder_award_id":"1115493","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7265051159","display_name":null,"funder_award_id":"IIS-1115493","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2611083268.pdf","grobid_xml":"https://content.openalex.org/works/W2611083268.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W1531492214","https://openalex.org/W1616462885","https://openalex.org/W1686810756","https://openalex.org/W1797268635","https://openalex.org/W1954152232","https://openalex.org/W1958236864","https://openalex.org/W1970381522","https://openalex.org/W1993309459","https://openalex.org/W2048012365","https://openalex.org/W2108598243","https://openalex.org/W2114720692","https://openalex.org/W2120396827","https://openalex.org/W2125943921","https://openalex.org/W2143890915","https://openalex.org/W2149273804","https://openalex.org/W2163605009","https://openalex.org/W2398018337","https://openalex.org/W3104998832"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"We":[0,91],"present":[1],"a":[2,21,30,45,68,77,86,93],"crowdsourcing":[3,96],"workflow":[4],"to":[5,38,98,126],"collect":[6],"image":[7],"annotations":[8],"for":[9],"visually":[10,101],"similar":[11,37,60],"synthetic":[12,53],"categories":[13,54,130],"without":[14],"requiring":[15],"experts.":[16,140],"In":[17],"animals,":[18],"there":[19],"is":[20],"direct":[22],"link":[23],"between":[24],"taxonomy":[25,61],"and":[26],"visual":[27,123],"similarity:":[28],"e.g.":[29,67],"collie":[31],"(type":[32],"of":[33,49,131,138],"dog)":[34],"looks":[35,73],"more":[36],"other":[39],"collies":[40],"(e.g.":[41],"smooth":[42],"collie)":[43],"than":[44],"greyhound":[46],"(another":[47],"type":[48],"dog).":[50],"However,":[51],"in":[52,119],"such":[55],"as":[56,76],"cars,":[57],"objects":[58,103],"with":[59,128],"can":[62],"have":[63],"very":[64,83],"different":[65,84],"appearance:":[66],"2011":[69,78,87],"Ford":[70,79,88],"F-150":[71,80,89],"Supercrew-HD":[72],"the":[74,120,136],"same":[75],"Supercrew-LL":[81],"but":[82],"from":[85],"Supercrew-SVT.":[90],"introduce":[92],"graph":[94],"based":[95],"algorithm":[97],"automatically":[99],"group":[100],"indistinguishable":[102],"together.":[104],"Using":[105],"our":[106],"workflow,":[107],"we":[108],"label":[109],"712,430":[110],"images":[111],"by":[112],"~1,000":[113],"Amazon":[114],"Mechanical":[115],"Turk":[116],"workers;":[117],"resulting":[118],"largest":[121],"fine-grained":[122],"dataset":[124],"reported":[125],"date":[127],"2,657":[129],"cars":[132],"annotated":[133],"at":[134],"1/20th":[135],"cost":[137],"hiring":[139]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}