{"id":"https://openalex.org/W2296739333","doi":"https://doi.org/10.1145/2835776.2835835","title":"Quality Management in Crowdsourcing using Gold Judges Behavior","display_name":"Quality Management in Crowdsourcing using Gold Judges Behavior","publication_year":2016,"publication_date":"2016-02-04","ids":{"openalex":"https://openalex.org/W2296739333","doi":"https://doi.org/10.1145/2835776.2835835","mag":"2296739333"},"language":"en","primary_location":{"id":"doi:10.1145/2835776.2835835","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2835776.2835835","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078735102","display_name":"Gabriella Kazai","orcid":"https://orcid.org/0009-0002-5158-6630"},"institutions":[{"id":"https://openalex.org/I4210162847","display_name":"Lumen (United Kingdom)","ror":"https://ror.org/05fexxx29","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210162847"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Gabriella Kazai","raw_affiliation_strings":["Lumi, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Lumi, London, United Kingdom","institution_ids":["https://openalex.org/I4210162847"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108365460","display_name":"Imed Zitouni","orcid":null},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Imed Zitouni","raw_affiliation_strings":["Microsoft, Bellevue, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Bellevue, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210108985"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5078735102"],"corresponding_institution_ids":["https://openalex.org/I4210162847"],"apc_list":null,"apc_paid":null,"fwci":16.5792,"has_fulltext":false,"cited_by_count":50,"citation_normalized_percentile":{"value":0.98656542,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"267","last_page":"276"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9837999939918518,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9733999967575073,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9736603498458862},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7302601337432861},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6711128354072571},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.6000226736068726},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.56903076171875},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.4629790186882019},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4480217695236206},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42068907618522644},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3746669292449951},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3239519000053406},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.15320417284965515},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11085650324821472}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9736603498458862},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7302601337432861},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6711128354072571},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.6000226736068726},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.56903076171875},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.4629790186882019},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4480217695236206},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42068907618522644},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3746669292449951},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3239519000053406},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.15320417284965515},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11085650324821472},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2835776.2835835","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2835776.2835835","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.6100000143051147,"id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W1525088415","https://openalex.org/W1541075651","https://openalex.org/W1590411898","https://openalex.org/W1678356000","https://openalex.org/W1755512443","https://openalex.org/W1970381522","https://openalex.org/W2003497265","https://openalex.org/W2005917549","https://openalex.org/W2008285050","https://openalex.org/W2024046085","https://openalex.org/W2026969101","https://openalex.org/W2094607109","https://openalex.org/W2098865355","https://openalex.org/W2104749423","https://openalex.org/W2109021302","https://openalex.org/W2116664070","https://openalex.org/W2139516750","https://openalex.org/W2149906572","https://openalex.org/W2151401338","https://openalex.org/W2165062189","https://openalex.org/W2167913131","https://openalex.org/W2170989440","https://openalex.org/W2187291759","https://openalex.org/W2251818274","https://openalex.org/W2315338392","https://openalex.org/W2949654880","https://openalex.org/W3121257585","https://openalex.org/W3121522380","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2178171640","https://openalex.org/W1472755691","https://openalex.org/W2055733113","https://openalex.org/W3102460643","https://openalex.org/W2522741018","https://openalex.org/W2109094787","https://openalex.org/W8975994","https://openalex.org/W2165062189","https://openalex.org/W2044507188","https://openalex.org/W4283738541"],"abstract_inverted_index":{"Crowdsourcing":[0],"relevance":[1],"labels":[2],"has":[3],"become":[4],"an":[5],"accepted":[6],"practice":[7],"for":[8,118],"the":[9,15,42,45,83,145,157,187],"evaluation":[10],"of":[11,17,27,44,140,147,189],"IR":[12],"systems,":[13],"where":[14],"task":[16,88],"constructing":[18],"a":[19,87,138,167],"test":[20],"collection":[21],"is":[22,48,80],"distributed":[23],"over":[24],"large":[25],"populations":[26],"unknown":[28],"users":[29],"with":[30,53,186],"widely":[31],"varied":[32],"skills":[33],"and":[34,40,72,89,151,154],"motivations.":[35],"Typical":[36],"methods":[37],"to":[38,49,70,81,124,165,169],"check":[39],"ensure":[41],"quality":[43,93,126],"crowd's":[46],"output":[47],"inject":[50],"work":[51,92],"tasks":[52,67,185],"known":[54],"answers":[55],"(gold":[56],"tasks)":[57],"on":[58,95],"which":[59,121],"workers'":[60,84],"performance":[61],"can":[62,114,122],"be":[63],"measured.":[64],"However,":[65],"gold":[66,105,163,190],"are":[68],"expensive":[69],"create":[71],"have":[73],"limited":[74],"application.":[75],"A":[76],"more":[77],"recent":[78],"trend":[79],"monitor":[82],"interactions":[85],"during":[86],"estimate":[90],"their":[91,96],"based":[94],"behavior.":[97],"In":[98],"this":[99],"paper,":[100],"we":[101,143],"show":[102,177],"that":[103,108,178],"without":[104],"behavior":[106,160,164,191],"signals":[107,161],"reflect":[109],"trusted":[110],"interaction":[111],"patterns,":[112],"classifiers":[113],"perform":[115],"poorly,":[116],"especially":[117],"complex":[119],"tasks,":[120],"lead":[123],"high":[125],"crowd":[127,152,173],"workers":[128,134,153],"getting":[129],"blocked":[130],"while":[131],"poorly":[132,171],"performing":[133,172],"remain":[135],"undetected.":[136],"Through":[137],"series":[139],"crowdsourcing":[141],"experiments,":[142],"compare":[144],"behaviors":[146],"trained":[148,158],"professional":[149],"judges":[150],"then":[155],"use":[156,188],"judges'":[159],"as":[162],"train":[166],"classifier":[168],"detect":[170],"workers.":[174],"Our":[175],"experiments":[176],"classification":[179],"accuracy":[180],"almost":[181],"doubles":[182],"in":[183],"some":[184],"data.":[192]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":9},{"year":2018,"cited_by_count":9},{"year":2017,"cited_by_count":8},{"year":2016,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}