{"id":"https://openalex.org/W2789102511","doi":"https://doi.org/10.1613/jair.5727","title":"Actively Estimating Crowd Annotation Consensus","display_name":"Actively Estimating Crowd Annotation Consensus","publication_year":2018,"publication_date":"2018-02-27","ids":{"openalex":"https://openalex.org/W2789102511","doi":"https://doi.org/10.1613/jair.5727","mag":"2789102511"},"language":"en","primary_location":{"id":"doi:10.1613/jair.5727","is_oa":true,"landing_page_url":"https://doi.org/10.1613/jair.5727","pdf_url":"https://jair.org/index.php/jair/article/download/11178/26383","source":{"id":"https://openalex.org/S139930977","display_name":"Journal of Artificial Intelligence Research","issn_l":"1076-9757","issn":["1076-9757","1943-5037"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310315760","host_organization_name":"AI Access Foundation","host_organization_lineage":["https://openalex.org/P4310315760"],"host_organization_lineage_names":["AI Access Foundation"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Artificial Intelligence Research","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://jair.org/index.php/jair/article/download/11178/26383","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111728155","display_name":"Yunus Emre Kara","orcid":null},"institutions":[{"id":"https://openalex.org/I4405392","display_name":"Bo\u011fazi\u00e7i University","ror":"https://ror.org/03z9tma90","country_code":"TR","type":"education","lineage":["https://openalex.org/I4405392"]}],"countries":["TR"],"is_corresponding":true,"raw_author_name":"Yunus Emre Kara","raw_affiliation_strings":["Department of Computer Engineering, Bogazici University TR-34342, Bebek, Istanbul, Turkey"],"affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Bogazici University TR-34342, Bebek, Istanbul, Turkey","institution_ids":["https://openalex.org/I4405392"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060285081","display_name":"Gaye Genc","orcid":null},"institutions":[{"id":"https://openalex.org/I4405392","display_name":"Bo\u011fazi\u00e7i University","ror":"https://ror.org/03z9tma90","country_code":"TR","type":"education","lineage":["https://openalex.org/I4405392"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Gaye Genc","raw_affiliation_strings":["Department of Computer Engineering, Bogazici University TR-34342, Bebek, Istanbul, Turkey"],"affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Bogazici University TR-34342, Bebek, Istanbul, Turkey","institution_ids":["https://openalex.org/I4405392"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076160972","display_name":"Oya Aran","orcid":"https://orcid.org/0000-0002-4679-9335"},"institutions":[{"id":"https://openalex.org/I4405392","display_name":"Bo\u011fazi\u00e7i University","ror":"https://ror.org/03z9tma90","country_code":"TR","type":"education","lineage":["https://openalex.org/I4405392"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Oya Aran","raw_affiliation_strings":["Department of Computer Engineering, Bogazici University TR-34342, Bebek, Istanbul, Turkey"],"affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Bogazici University TR-34342, Bebek, Istanbul, Turkey","institution_ids":["https://openalex.org/I4405392"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064788653","display_name":"Lale Akarun","orcid":"https://orcid.org/0000-0002-8813-8084"},"institutions":[{"id":"https://openalex.org/I4405392","display_name":"Bo\u011fazi\u00e7i University","ror":"https://ror.org/03z9tma90","country_code":"TR","type":"education","lineage":["https://openalex.org/I4405392"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Lale Akarun","raw_affiliation_strings":["Department of Computer Engineering, Bogazici University TR-34342, Bebek, Istanbul, Turkey"],"affiliations":[{"raw_affiliation_string":"Department of Computer Engineering, Bogazici University TR-34342, Bebek, Istanbul, Turkey","institution_ids":["https://openalex.org/I4405392"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5111728155"],"corresponding_institution_ids":["https://openalex.org/I4405392"],"apc_list":null,"apc_paid":null,"fwci":1.2605,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.82443055,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"61","issue":null,"first_page":"363","last_page":"405"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11598","display_name":"Internet Traffic Analysis and Secure E-voting","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9764999747276306,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8479524850845337},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7106847763061523},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.7086305618286133},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6627523303031921},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5979754328727722},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5575853586196899},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.522057056427002},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5168700218200684},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.5077370405197144},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5048232674598694},{"id":"https://openalex.org/keywords/crowd-sourcing","display_name":"Crowd sourcing","score":0.43948400020599365},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.42269569635391235},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.36747729778289795},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.36210083961486816},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.12577000260353088}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8479524850845337},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7106847763061523},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.7086305618286133},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6627523303031921},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5979754328727722},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5575853586196899},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.522057056427002},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5168700218200684},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.5077370405197144},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5048232674598694},{"id":"https://openalex.org/C3018396927","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowd sourcing","level":2,"score":0.43948400020599365},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.42269569635391235},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36747729778289795},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36210083961486816},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.12577000260353088},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1613/jair.5727","is_oa":true,"landing_page_url":"https://doi.org/10.1613/jair.5727","pdf_url":"https://jair.org/index.php/jair/article/download/11178/26383","source":{"id":"https://openalex.org/S139930977","display_name":"Journal of Artificial Intelligence Research","issn_l":"1076-9757","issn":["1076-9757","1943-5037"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310315760","host_organization_name":"AI Access Foundation","host_organization_lineage":["https://openalex.org/P4310315760"],"host_organization_lineage_names":["AI Access Foundation"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Artificial Intelligence Research","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1613/jair.5727","is_oa":true,"landing_page_url":"https://doi.org/10.1613/jair.5727","pdf_url":"https://jair.org/index.php/jair/article/download/11178/26383","source":{"id":"https://openalex.org/S139930977","display_name":"Journal of Artificial Intelligence Research","issn_l":"1076-9757","issn":["1076-9757","1943-5037"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310315760","host_organization_name":"AI Access Foundation","host_organization_lineage":["https://openalex.org/P4310315760"],"host_organization_lineage_names":["AI Access Foundation"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Artificial Intelligence Research","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2789102511.pdf","grobid_xml":"https://content.openalex.org/works/W2789102511.grobid-xml"},"referenced_works_count":49,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1497882404","https://openalex.org/W1542068205","https://openalex.org/W1601808502","https://openalex.org/W1703951723","https://openalex.org/W1782953462","https://openalex.org/W1818395637","https://openalex.org/W2021732807","https://openalex.org/W2042627297","https://openalex.org/W2058892254","https://openalex.org/W2085999189","https://openalex.org/W2109021302","https://openalex.org/W2115206325","https://openalex.org/W2116664070","https://openalex.org/W2121200917","https://openalex.org/W2125943921","https://openalex.org/W2137935418","https://openalex.org/W2140198703","https://openalex.org/W2140890285","https://openalex.org/W2149906572","https://openalex.org/W2163522723","https://openalex.org/W2164124780","https://openalex.org/W2167460663","https://openalex.org/W2174985112","https://openalex.org/W2180771322","https://openalex.org/W2181558882","https://openalex.org/W2197945395","https://openalex.org/W2258849743","https://openalex.org/W2326254517","https://openalex.org/W2553923866","https://openalex.org/W2579642215","https://openalex.org/W2903158431","https://openalex.org/W2949107948","https://openalex.org/W2963606337","https://openalex.org/W4239072543","https://openalex.org/W4300115751","https://openalex.org/W4394646033","https://openalex.org/W6638400380","https://openalex.org/W6655840284","https://openalex.org/W6660947052","https://openalex.org/W6676297131","https://openalex.org/W6679959949","https://openalex.org/W6685936606","https://openalex.org/W6687612953","https://openalex.org/W6692184408","https://openalex.org/W6732730354","https://openalex.org/W6756615331","https://openalex.org/W7005846345","https://openalex.org/W7062008220"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W2409650238","https://openalex.org/W2524552836","https://openalex.org/W2039747859","https://openalex.org/W2002094063","https://openalex.org/W2483845706","https://openalex.org/W2572825458","https://openalex.org/W2467989257","https://openalex.org/W2610818984","https://openalex.org/W2976765013"],"abstract_inverted_index":{"The":[0],"rapid":[1],"growth":[2],"of":[3,21,41,46,58,76,92,101,113,174,222,270],"storage":[4],"capacity":[5],"and":[6,34,65,74,96,108,162,189,218,250,280],"processing":[7],"power":[8],"has":[9,25,52],"caused":[10],"machine":[11],"learning":[12],"applications":[13],"to":[14,31,37,60,71,106,167,283],"increasingly":[15],"rely":[16],"on":[17,156,228,242],"using":[18,264],"immense":[19],"amounts":[20,40],"labeled":[22],"data.":[23,42],"It":[24],"become":[26],"more":[27],"important":[28],"than":[29],"ever":[30],"have":[32],"fast":[33],"inexpensive":[35],"ways":[36],"annotate":[38],"vast":[39],"With":[43],"the":[44,49,56,72,77,93,97,123,172,186,208,216,220,234,243,248,251,271,284],"emergence":[45],"crowdsourcing":[47],"services,":[48],"research":[50],"direction":[51],"gravitated":[53],"toward":[54],"putting":[55],"wisdom":[57],"crowds":[59],"better":[61],"use.":[62],"Unfortunately,":[63],"spammers":[64],"inattentive":[66],"annotators":[67,214],"pose":[68],"a":[69,89,102,169,200,224,278],"threat":[70],"quality":[73,81],"trustworthiness":[75],"consensus.":[78],"Thus,":[79],"high":[80],"consensus":[82,147,170,229,262],"estimation":[83],"from":[84,148],"crowd":[85,150],"annotated":[86],"data":[87],"requires":[88],"meticulous":[90],"choice":[91,118],"candidate":[94,187],"annotator":[95,157,188,205],"sample":[98,190],"in":[99,127,171,247],"need":[100],"new":[103,213],"annotation.":[104,195],"Due":[105],"time":[107],"budget":[109,279],"limitations,":[110],"it":[111],"is":[112,119,126,154,165],"utmost":[114],"importance":[115],"that":[116,257],"this":[117,131,136],"carried":[120],"out":[121],"while":[122],"annotation":[124],"collection":[125],"progress.":[128],"We":[129,180,231],"call":[130],"process":[132],"active":[133,141],"crowd-labeling.":[134],"To":[135],"end,":[137],"we":[138,198],"propose":[139,199],"an":[140,194],"crowd-labeling":[142,285],"approach":[143],"for":[144,184,192,203,211],"actively":[145],"estimating":[146],"continuous-valued":[149],"annotations.":[151],"Our":[152],"method":[153,202,259],"based":[155,227],"models":[158],"with":[159],"unknown":[160],"parameters,":[161],"Bayesian":[163],"inference":[164],"employed":[166],"reach":[168],"form":[173],"ordinal,":[175],"binary,":[176],"or":[177],"continuous":[178],"values.":[179],"introduce":[181,233],"ranking":[182],"functions":[183],"choosing":[185],"pair":[191],"requesting":[193],"In":[196],"addition,":[197],"penalizing":[201],"preventing":[204],"domination,":[206],"investigate":[207],"explore-exploit":[209],"trade-off":[210],"incorporating":[212],"into":[215],"system,":[217],"study":[219],"effects":[221],"inducing":[223],"stopping":[225],"criterion":[226],"quality.":[230],"also":[232],"crowd-labeled":[235],"Head":[236,252],"Pose":[237,253],"Annotations":[238,254],"datasets.":[239],"Experimental":[240],"results":[241],"benchmark":[244],"datasets":[245,255],"used":[246],"literature":[249],"suggest":[256],"our":[258],"provides":[260],"high-quality":[261],"by":[263],"as":[265,267],"few":[266],"one":[268],"fifth":[269],"annotations":[272],"(~80%":[273],"cost":[274],"reduction),":[275],"thereby":[276],"providing":[277],"time-sensitive":[281],"solution":[282],"problem.":[286]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}