{"id":"https://openalex.org/W2807747798","doi":"https://doi.org/10.65109/mmjs7280","title":"CrowdEval: A Cost-Efficient Strategy to Evaluate Crowdsourced Worker's Reliability","display_name":"CrowdEval: A Cost-Efficient Strategy to Evaluate Crowdsourced Worker's Reliability","publication_year":2018,"publication_date":"2018-07-09","ids":{"openalex":"https://openalex.org/W2807747798","doi":"https://doi.org/10.65109/mmjs7280","mag":"2807747798"},"language":"en","primary_location":{"id":"doi:10.65109/mmjs7280","doi":"https://doi.org/10.65109/mmjs7280","is_oa":false,"landing_page_url":"https://doi.org/10.65109/mmjs7280","pdf_url":null,"source":{"id":"https://openalex.org/S4306417597","display_name":"Adaptive Agents and Multi-Agents Systems","issn_l":"1548-8403","issn":["1548-8403"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Joint Conference on Autonomous Agents and Multiagent Systems","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070496959","display_name":"Chenxi Qiu","orcid":"https://orcid.org/0000-0002-5288-4050","observed_orcids":["https://orcid.org/0000-0002-5288-4050"]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chenxi Qiu","raw_affiliation_strings":["Pennsylvania State University, University Park, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pennsylvania State University, University Park, PA, USA","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062519505","display_name":"Anna Squicciarini","orcid":"https://orcid.org/0000-0002-7396-1895","observed_orcids":["https://orcid.org/0000-0002-7396-1895"]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anna Squicciarini","raw_affiliation_strings":["Pennsylvania State University, University Park, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pennsylvania State University, University Park, PA, USA","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084521330","display_name":"Dev Rishi Khare","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dev Rishi Khare","raw_affiliation_strings":["Pennsylvania State University, University Park, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pennsylvania State University, University Park, PA, USA","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057684417","display_name":"Barbara Carminati","orcid":"https://orcid.org/0000-0002-7502-4731","observed_orcids":["https://orcid.org/0000-0002-7502-4731"]},"institutions":[{"id":"https://openalex.org/I115752224","display_name":"University of Insubria","ror":"https://ror.org/00s409261","country_code":"IT","type":"education","lineage":["https://openalex.org/I115752224"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Barbara Carminati","raw_affiliation_strings":["University of Insubria, Varese, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Insubria, Varese, Italy","institution_ids":["https://openalex.org/I115752224"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048489384","display_name":"James Caverlee","orcid":"https://orcid.org/0000-0001-8350-8528","observed_orcids":["https://orcid.org/0000-0001-8350-8528"]},"institutions":[{"id":"https://openalex.org/I91045830","display_name":"Texas A&M University","ror":"https://ror.org/01f5ytq51","country_code":"US","type":"education","lineage":["https://openalex.org/I91045830"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Caverlee","raw_affiliation_strings":["Texas A&M University, College Station, TX, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Texas A&M University, College Station, TX, USA","institution_ids":["https://openalex.org/I91045830"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.319,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.88611926,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1486","last_page":"1494"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.6549999713897705},{"id":"https://openalex.org/keywords/employee-reliability","display_name":"worker reliability","score":0.9079999923706055},{"id":"https://openalex.org/keywords/worker-evaluation","display_name":"worker evaluation","score":0.48100000619888306},{"id":"https://openalex.org/keywords/strategic-behavior","display_name":"strategic behavior","score":0.6309999823570251},{"id":"https://openalex.org/keywords/malicious-workers","display_name":"malicious workers","score":0.9440000057220459},{"id":"https://openalex.org/keywords/gold-standard-evaluation","display_name":"gold standard evaluation","score":0.8159999847412109},{"id":"https://openalex.org/keywords/amazon-mechanical-turk","display_name":"Amazon Mechanical Turk","score":0.8510000109672546}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8239006996154785},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7752470970153809},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7162777185440063},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6116139888763428},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5975317358970642},{"id":"https://openalex.org/C2778139618","wikidata":"https://www.wikidata.org/wiki/Q13440398","display_name":"Workforce","level":2,"score":0.5393436551094055},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5152445435523987},{"id":"https://openalex.org/C2775936607","wikidata":"https://www.wikidata.org/wiki/Q466845","display_name":"Tracking (education)","level":2,"score":0.5128998756408691},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4838823080062866},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.4380662441253662},{"id":"https://openalex.org/C11644782","wikidata":"https://www.wikidata.org/wiki/Q15401790","display_name":"Cost efficiency","level":2,"score":0.41591835021972656},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28395161032676697},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24841362237930298},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.14439398050308228},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09702780842781067},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09181469678878784},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.65109/mmjs7280","doi":"https://doi.org/10.65109/mmjs7280","is_oa":false,"landing_page_url":"https://doi.org/10.65109/mmjs7280","pdf_url":null,"source":{"id":"https://openalex.org/S4306417597","display_name":"Adaptive Agents and Multi-Agents Systems","issn_l":"1548-8403","issn":["1548-8403"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Joint Conference on Autonomous Agents and Multiagent Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:irinsubria.uninsubria.it:11383/2095836","doi":null,"is_oa":false,"landing_page_url":"http://hdl.handle.net/11383/2095836","pdf_url":null,"source":{"id":"https://openalex.org/S4377196351","display_name":"IrInSubria (University of Insubria)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I115752224","host_organization_name":"University of Insubria","host_organization_lineage":["https://openalex.org/I115752224"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"mag:2807747798","doi":null,"is_oa":false,"landing_page_url":"https://dblp.uni-trier.de/db/conf/atal/aamas2018.html#QiuSKCC18","pdf_url":null,"source":{"id":"https://openalex.org/S4306417597","display_name":"Adaptive Agents and Multi-Agents Systems","issn_l":"1548-8403","issn":["1548-8403"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Adaptive Agents and Multi-Agents Systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W279631868","https://openalex.org/W1497882404","https://openalex.org/W1882350379","https://openalex.org/W1943309769","https://openalex.org/W1976617111","https://openalex.org/W1997504572","https://openalex.org/W2091004778","https://openalex.org/W2121117737","https://openalex.org/W2140890285","https://openalex.org/W2142518823","https://openalex.org/W2147687736","https://openalex.org/W2149489787","https://openalex.org/W2163522723","https://openalex.org/W2294346744","https://openalex.org/W2296319761","https://openalex.org/W2533800767","https://openalex.org/W2538903535","https://openalex.org/W2736287601"],"related_works":["https://openalex.org/W2078503914","https://openalex.org/W2584427494","https://openalex.org/W7099142948","https://openalex.org/W2401033271","https://openalex.org/W2538903535","https://openalex.org/W1872614887","https://openalex.org/W2794325666","https://openalex.org/W2163522723","https://openalex.org/W3169854180","https://openalex.org/W2614452370"],"abstract_inverted_index":{"Crowdsourcing":[0],"platforms":[1],"depend":[2],"on":[3],"the":[4,20,27,68,71,78],"quality":[5],"of":[6,22,29,70,74,81,108],"work":[7],"provided":[8],"by":[9],"a":[10,39,103],"distributed":[11],"workforce.":[12],"Yet,":[13],"it":[14],"is":[15],"challenging":[16],"to":[17,44,61,77,113],"dependably":[18],"measure":[19,62],"reliability":[21],"these":[23],"workers,":[24],"particularly":[25],"in":[26,106],"face":[28],"strategic":[30],"or":[31,87],"malicious":[32],"behavior.":[33],"In":[34,49],"this":[35],"paper,":[36],"we":[37,51,97],"present":[38],"dynamic":[40],"and":[41,57,66,110],"efficient":[42],"solution":[43],"keep":[45],"tracking":[46],"workers'":[47,82],"reliability.":[48],"particular,":[50],"use":[52],"both":[53],"gold":[54],"standard":[55],"evaluation":[56,60,75],"peer":[58],"consistency":[59],"each":[63],"worker":[64],"performance,":[65],"adjust":[67],"proportion":[69],"two":[72],"types":[73],"according":[76],"estimated":[79],"distribution":[80],"behavior":[83],"(e.g.,":[84],"being":[85],"reliable":[86],"malicious).":[88],"Through":[89],"experiments":[90],"over":[91],"real":[92],"Amazon":[93],"Mechanical":[94],"Turk":[95],"traces,":[96],"find":[98],"that":[99],"our":[100],"approach":[101],"has":[102],"significant":[104],"gain":[105],"terms":[107],"accuracy":[109],"cost":[111],"compared":[112],"state-of-the-art":[114],"algorithms.":[115]},"counts_by_year":[{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2018-06-21T00:00:00"}