{"id":"https://openalex.org/W2967708711","doi":"https://doi.org/10.1145/3237188","title":"FourEyes","display_name":"FourEyes","publication_year":2019,"publication_date":"2019-08-09","ids":{"openalex":"https://openalex.org/W2967708711","doi":"https://doi.org/10.1145/3237188","mag":"2967708711"},"language":"en","primary_location":{"id":"doi:10.1145/3237188","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3237188","pdf_url":null,"source":{"id":"https://openalex.org/S4210173818","display_name":"ACM Transactions on Interactive Intelligent Systems","issn_l":"2160-6455","issn":["2160-6455","2160-6463"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Interactive Intelligent Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058977902","display_name":"Jean Song","orcid":"https://orcid.org/0000-0003-4379-3971"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jean Y. Song","raw_affiliation_strings":["University of Michigan, Ann Arbor, MI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor, MI, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066433489","display_name":"Raymond Fok","orcid":"https://orcid.org/0000-0002-0612-5782"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Raymond Fok","raw_affiliation_strings":["University of Michigan, Ann Arbor, MI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor, MI, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079709359","display_name":"Juho Kim","orcid":"https://orcid.org/0000-0001-6348-4127"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Juho Kim","raw_affiliation_strings":["KAIST, Republic of Korea, Daejeon, Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KAIST, Republic of Korea, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063574664","display_name":"Walter S. Lasecki","orcid":null},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Walter S. Lasecki","raw_affiliation_strings":["University of Michigan, Ann Arbor, MI, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor, MI, USA","institution_ids":["https://openalex.org/I27837315"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9374,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.80728982,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"10","issue":"1","first_page":"1","last_page":"30"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9696999788284302,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9220238327980042},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7866878509521484},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.7708569765090942},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.706551194190979},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5599908828735352},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4787680208683014},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.47741615772247314},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4505902826786041},{"id":"https://openalex.org/keywords/image-segmentation","display_name":"Image segmentation","score":0.4279898703098297},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.42752671241760254},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.35366883873939514}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9220238327980042},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7866878509521484},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.7708569765090942},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.706551194190979},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5599908828735352},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4787680208683014},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.47741615772247314},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4505902826786041},{"id":"https://openalex.org/C124504099","wikidata":"https://www.wikidata.org/wiki/Q56933","display_name":"Image segmentation","level":3,"score":0.4279898703098297},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.42752671241760254},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35366883873939514},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3237188","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3237188","pdf_url":null,"source":{"id":"https://openalex.org/S4210173818","display_name":"ACM Transactions on Interactive Intelligent Systems","issn_l":"2160-6455","issn":["2160-6455","2160-6463"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Interactive Intelligent Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W611457968","https://openalex.org/W1524782856","https://openalex.org/W1534477342","https://openalex.org/W1565732937","https://openalex.org/W1573900212","https://openalex.org/W1861492603","https://openalex.org/W1903029394","https://openalex.org/W1968326021","https://openalex.org/W1970381522","https://openalex.org/W1988690348","https://openalex.org/W1988790447","https://openalex.org/W1989563738","https://openalex.org/W2005665592","https://openalex.org/W2031356414","https://openalex.org/W2035818959","https://openalex.org/W2045515072","https://openalex.org/W2055302526","https://openalex.org/W2058179030","https://openalex.org/W2060903226","https://openalex.org/W2088664555","https://openalex.org/W2098865355","https://openalex.org/W2102911672","https://openalex.org/W2110764733","https://openalex.org/W2124994029","https://openalex.org/W2127008633","https://openalex.org/W2133485007","https://openalex.org/W2135293965","https://openalex.org/W2138847321","https://openalex.org/W2142518823","https://openalex.org/W2146483860","https://openalex.org/W2148479118","https://openalex.org/W2149273804","https://openalex.org/W2168765606","https://openalex.org/W2173894669","https://openalex.org/W2194370442","https://openalex.org/W2264285263","https://openalex.org/W2280438363","https://openalex.org/W2336395599","https://openalex.org/W2337429362","https://openalex.org/W2505156521","https://openalex.org/W2591957553","https://openalex.org/W2603933151","https://openalex.org/W2611083268","https://openalex.org/W2765966685","https://openalex.org/W2793887949","https://openalex.org/W2794122052","https://openalex.org/W2795487395","https://openalex.org/W2796095752","https://openalex.org/W2796171214","https://openalex.org/W2810719472","https://openalex.org/W2953080935","https://openalex.org/W2963150697","https://openalex.org/W4230939871"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114","https://openalex.org/W1522196789"],"abstract_inverted_index":{"Crowdsourcing":[0],"is":[1,27,112],"a":[2,14,94,103,115,157,171,209,267,277],"common":[3],"means":[4,104],"of":[5,16,105,118,145,173,196,204,232,237,261,286],"collecting":[6],"image":[7,24,35,159,219],"segmentation":[8,25,51,160,220],"training":[9],"data":[10],"for":[11,199,213,218],"use":[12],"in":[13,33,81,202],"variety":[15],"computer":[17],"vision":[18],"applications.":[19],"However,":[20,68],"designing":[21],"accurate":[22,66,216],"crowd-powered":[23,158],"systems":[26],"challenging,":[28],"because":[29],"defining":[30],"object":[31],"boundaries":[32],"an":[34,238],"requires":[36],"significant":[37],"fine":[38],"motor":[39],"skills":[40],"and":[41,55,75,154,180,282],"hand-eye":[42],"coordination,":[43],"which":[44],"makes":[45],"these":[46],"tasks":[47,221],"error-prone.":[48],"Typically,":[49],"special":[50],"tools":[52,124,139,153,185],"are":[53,61],"created":[54],"then":[56,169],"answers":[57],"from":[58],"multiple":[59,184],"workers":[60],"aggregated":[62],"to":[63],"generate":[64],"more":[65,215],"results.":[67],"individual":[69,138,249],"tool":[70,100],"designs":[71],"can":[72,125,186,257],"bias":[73],"how":[74],"where":[76],"people":[77],"make":[78],"mistakes,":[79],"resulting":[80],"shared":[82],"errors":[83],"that":[84,98,113,162,175,182,245],"remain":[85],"even":[86],"after":[87],"aggregation.":[88,263],"In":[89,226],"this":[90],"article,":[91],"we":[92,149,192,228],"introduce":[93,208],"novel":[95,210,278],"crowdsourcing":[96],"approach":[97],"leverages":[99],"diversity":[101],"as":[102],"improving":[106],"aggregate":[107,189],"crowd":[108],"performance.":[109],"Our":[110,242],"idea":[111,195],"given":[114],"diverse":[116],"set":[117],"tools,":[119],"answer":[120],"aggregation":[121,164,178,201,240],"done":[122],"across":[123,165],"help":[126],"improve":[127,188],"the":[128,137,143,146,194,230,234,248,254,259,287],"collective":[129],"performance":[130,260],"by":[131,136,275],"offsetting":[132],"systematic":[133],"biases":[134],"induced":[135],"themselves.":[140],"To":[141],"demonstrate":[142],"effectiveness":[144],"proposed":[147,288],"approach,":[148],"design":[150],"four":[151],"different":[152,166,177],"present":[155],"FourEyes,":[156],"system":[161],"uses":[163],"tools.":[167],"We":[168,207],"conduct":[170],"series":[172],"studies":[174],"evaluate":[176],"conditions":[179],"show":[181],"using":[183],"significantly":[187],"accuracy.":[190],"Furthermore,":[191],"investigate":[193],"applying":[197],"post-processing":[198],"multi-tool":[200,262],"terms":[203],"correction":[205,255],"mechanism.":[206],"region-based":[211,279],"method":[212,281],"synthesizing":[214],"bounds":[217],"through":[222],"averaging":[223],"surrounding":[224],"annotations.":[225],"addition,":[227],"explore":[229],"effect":[231],"adjusting":[233],"threshold":[235],"parameter":[236],"EM-based":[239],"method.":[241],"results":[243],"suggest":[244],"not":[246],"only":[247],"tool\u2019s":[250],"design,":[251],"but":[252],"also":[253],"mechanism,":[256],"affect":[258],"This":[264],"article":[265],"extends":[266],"work":[268],"presented":[269],"at":[270],"ACM":[271],"IUI":[272],"2018":[273],"[46]":[274],"providing":[276],"error-correction":[280],"additional":[283],"in-depth":[284],"evaluation":[285],"approach.":[289]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2019-08-22T00:00:00"}