{"id":"https://openalex.org/W2595576831","doi":"https://doi.org/10.1609/hcomp.v4i1.13293","title":"Interactive Consensus Agreement Games for Labeling Images","display_name":"Interactive Consensus Agreement Games for Labeling Images","publication_year":2016,"publication_date":"2016-09-21","ids":{"openalex":"https://openalex.org/W2595576831","doi":"https://doi.org/10.1609/hcomp.v4i1.13293","mag":"2595576831"},"language":"en","primary_location":{"id":"doi:10.1609/hcomp.v4i1.13293","is_oa":true,"landing_page_url":"https://doi.org/10.1609/hcomp.v4i1.13293","pdf_url":"https://ojs.aaai.org/index.php/HCOMP/article/download/13293/13141","source":{"id":"https://openalex.org/S4387284122","display_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","issn_l":"2769-1330","issn":["2769-1330","2769-1349"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/HCOMP/article/download/13293/13141","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034095010","display_name":"Paul Upchurch","orcid":"https://orcid.org/0000-0001-9293-7367"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Paul Upchurch","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070586406","display_name":"Daniel Sedra","orcid":null},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Sedra","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024651979","display_name":"Andrew Mullen","orcid":"https://orcid.org/0000-0002-2819-4797"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Mullen","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089280215","display_name":"Haym Hirsh","orcid":null},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haym Hirsh","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087263986","display_name":"Kavita Bala","orcid":"https://orcid.org/0000-0001-9761-6503"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kavita Bala","raw_affiliation_strings":["Cornell University"],"affiliations":[{"raw_affiliation_string":"Cornell University","institution_ids":["https://openalex.org/I205783295"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5034095010"],"corresponding_institution_ids":["https://openalex.org/I205783295"],"apc_list":null,"apc_paid":null,"fwci":2.7631,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.92621883,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"4","issue":null,"first_page":"239","last_page":"248"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.835956335067749},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.6808160543441772},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.6603768467903137},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5977092385292053},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.548832893371582},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4309206008911133},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4174236059188843},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.41171014308929443},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.349470317363739},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.32475972175598145}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.835956335067749},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.6808160543441772},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6603768467903137},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5977092385292053},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.548832893371582},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4309206008911133},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4174236059188843},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.41171014308929443},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.349470317363739},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32475972175598145},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1609/hcomp.v4i1.13293","is_oa":true,"landing_page_url":"https://doi.org/10.1609/hcomp.v4i1.13293","pdf_url":"https://ojs.aaai.org/index.php/HCOMP/article/download/13293/13141","source":{"id":"https://openalex.org/S4387284122","display_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","issn_l":"2769-1330","issn":["2769-1330","2769-1349"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1609/hcomp.v4i1.13293","is_oa":true,"landing_page_url":"https://doi.org/10.1609/hcomp.v4i1.13293","pdf_url":"https://ojs.aaai.org/index.php/HCOMP/article/download/13293/13141","source":{"id":"https://openalex.org/S4387284122","display_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","issn_l":"2769-1330","issn":["2769-1330","2769-1349"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.4699999988079071,"id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2595576831.pdf","grobid_xml":"https://content.openalex.org/works/W2595576831.grobid-xml"},"referenced_works_count":82,"referenced_works":["https://openalex.org/W80047990","https://openalex.org/W323338930","https://openalex.org/W581913467","https://openalex.org/W972416066","https://openalex.org/W1494630173","https://openalex.org/W1558395318","https://openalex.org/W1576013900","https://openalex.org/W1580717584","https://openalex.org/W1584070893","https://openalex.org/W1595524969","https://openalex.org/W1600836884","https://openalex.org/W1603672842","https://openalex.org/W1665651668","https://openalex.org/W1839367233","https://openalex.org/W1946367136","https://openalex.org/W1953465585","https://openalex.org/W1970381522","https://openalex.org/W1975339897","https://openalex.org/W1978113971","https://openalex.org/W1984465233","https://openalex.org/W1997504572","https://openalex.org/W2012215691","https://openalex.org/W2024559213","https://openalex.org/W2034692547","https://openalex.org/W2042632603","https://openalex.org/W2055302526","https://openalex.org/W2060376762","https://openalex.org/W2065010255","https://openalex.org/W2073598993","https://openalex.org/W2076022814","https://openalex.org/W2079140531","https://openalex.org/W2085989833","https://openalex.org/W2108598243","https://openalex.org/W2110764733","https://openalex.org/W2112193291","https://openalex.org/W2120898858","https://openalex.org/W2125943921","https://openalex.org/W2134670479","https://openalex.org/W2137935418","https://openalex.org/W2141282920","https://openalex.org/W2142897866","https://openalex.org/W2147004922","https://openalex.org/W2147687736","https://openalex.org/W2149489787","https://openalex.org/W2154262598","https://openalex.org/W2156677648","https://openalex.org/W2158880898","https://openalex.org/W2163605009","https://openalex.org/W2168396085","https://openalex.org/W2168765606","https://openalex.org/W2170016732","https://openalex.org/W2184559750","https://openalex.org/W2267283990","https://openalex.org/W2271622619","https://openalex.org/W2340840968","https://openalex.org/W2404374285","https://openalex.org/W2724747146","https://openalex.org/W2728944333","https://openalex.org/W2793732086","https://openalex.org/W2952598059","https://openalex.org/W2963204789","https://openalex.org/W3001610306","https://openalex.org/W3098309574","https://openalex.org/W4239201551","https://openalex.org/W4253646065","https://openalex.org/W6616890673","https://openalex.org/W6625690124","https://openalex.org/W6635741605","https://openalex.org/W6636232879","https://openalex.org/W6644888336","https://openalex.org/W6646276392","https://openalex.org/W6659291473","https://openalex.org/W6665346890","https://openalex.org/W6666817399","https://openalex.org/W6669389602","https://openalex.org/W6670083791","https://openalex.org/W6676297131","https://openalex.org/W6684687462","https://openalex.org/W6684916453","https://openalex.org/W6685220563","https://openalex.org/W6749519847","https://openalex.org/W6766110196"],"related_works":["https://openalex.org/W4312417841","https://openalex.org/W4321369474","https://openalex.org/W2731899572","https://openalex.org/W3133861977","https://openalex.org/W4200173597","https://openalex.org/W3116150086","https://openalex.org/W2999805992","https://openalex.org/W4380075502","https://openalex.org/W4291897433","https://openalex.org/W4223943233"],"abstract_inverted_index":{"Scene":[0],"understanding":[1],"algorithms":[2,34],"in":[3,87],"computer":[4],"vision":[5],"are":[6,35,47,93],"improving":[7],"dramatically":[8],"by":[9,50,101,119,137],"training":[10,28,184],"deep":[11],"convolutional":[12],"neural":[13],"networks":[14],"on":[15,139,166],"millions":[16],"of":[17,27,69,122,125,128,146],"accurately":[18,177],"annotated":[19],"images.":[20],"Collecting":[21],"large-scale":[22],"datasets":[23,142,185],"for":[24,114,186],"this":[25,133],"kind":[26],"is":[29,85,155],"challenging,":[30],"and":[31,180],"the":[32,40,52,67,70,76,120,167],"learning":[33,187],"only":[36],"as":[37,39,62],"good":[38],"data":[41],"they":[42],"train":[43],"on.":[44],"Training":[45],"annotations":[46,72,92,116],"often":[48],"obtained":[49,100],"taking":[51],"majority":[53],"label":[54],"from":[55],"independent":[56,90],"crowdsourced":[57],"workers":[58,103],"using":[59],"platforms":[60],"such":[61],"Amazon":[63],"Mechanical":[64],"Turk.":[65],"However,":[66],"accuracy":[68],"resulting":[71],"can":[73,98],"vary,":[74],"with":[75],"hardest-to-annotate":[77],"samples":[78],"having":[79,102],"prohibitively":[80],"low":[81],"accuracy.":[82],"Our":[83],"insight":[84],"that":[86,132],"cases":[88],"where":[89],"worker":[91],"poor":[94],"more":[95,176,182],"accurate":[96],"results":[97],"be":[99,164],"collaborate.":[104],"This":[105],"paper":[106],"introduces":[107],"consensus":[108],"agreement":[109,121],"games,":[110],"a":[111,144],"novel":[112],"method":[113,159,172],"assigning":[115],"to":[117,163,175],"images":[118,179],"multiple":[123],"consensuses":[124],"small":[126],"cliques":[127],"workers.":[129],"We":[130],"demonstrate":[131],"approach":[134],"reduces":[135],"error":[136],"37.8%":[138],"two":[140],"different":[141],"at":[143],"cost":[145,154],"$0.10":[147],"or":[148],"$0.17":[149],"per":[150],"annotation.":[151],"The":[152],"higher":[153],"justified":[156],"because":[157],"our":[158,171],"does":[160],"not":[161],"need":[162],"run":[165],"entire":[168],"dataset.":[169],"Ultimately,":[170],"enables":[173],"us":[174],"annotate":[178],"build":[181],"challenging":[183],"algorithms.":[188]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2017,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}