{"id":"https://openalex.org/W2292984643","doi":"https://doi.org/10.21437/interspeech.2011-764","title":"Crowdsourcing preference tests, and how to detect cheating","display_name":"Crowdsourcing preference tests, and how to detect cheating","publication_year":2011,"publication_date":"2011-08-27","ids":{"openalex":"https://openalex.org/W2292984643","doi":"https://doi.org/10.21437/interspeech.2011-764","mag":"2292984643"},"language":"en","primary_location":{"id":"doi:10.21437/interspeech.2011-764","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2011-764","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2011","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004649927","display_name":"Sabine Buchholz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sabine Buchholz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5071418947","display_name":"Javier Latorre","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Javier Latorre","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":96,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3053","last_page":"3056"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9592000246047974,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10326","display_name":"Indoor and Outdoor Localization Technologies","score":0.958299994468689,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cheating","display_name":"Cheating","score":0.9664936065673828},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9408177137374878},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.7394049763679504},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5843114852905273},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5624449849128723},{"id":"https://openalex.org/keywords/active-listening","display_name":"Active listening","score":0.4496493935585022},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.41747161746025085},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41271042823791504},{"id":"https://openalex.org/keywords/index","display_name":"Index (typography)","score":0.41075050830841064},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3251626193523407},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.21711212396621704},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.16018453240394592},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.11662071943283081},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.11298713088035583},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11011350154876709},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.06392031908035278}],"concepts":[{"id":"https://openalex.org/C2778024590","wikidata":"https://www.wikidata.org/wiki/Q2357432","display_name":"Cheating","level":2,"score":0.9664936065673828},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9408177137374878},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.7394049763679504},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5843114852905273},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5624449849128723},{"id":"https://openalex.org/C177291462","wikidata":"https://www.wikidata.org/wiki/Q423038","display_name":"Active listening","level":2,"score":0.4496493935585022},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.41747161746025085},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41271042823791504},{"id":"https://openalex.org/C2777382242","wikidata":"https://www.wikidata.org/wiki/Q6017816","display_name":"Index (typography)","level":2,"score":0.41075050830841064},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3251626193523407},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.21711212396621704},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.16018453240394592},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.11662071943283081},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11298713088035583},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11011350154876709},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.06392031908035278},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.21437/interspeech.2011-764","is_oa":false,"landing_page_url":"https://doi.org/10.21437/interspeech.2011-764","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Interspeech 2011","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.8100000023841858,"display_name":"Reduced inequalities"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W58567859","https://openalex.org/W70888257","https://openalex.org/W1539053441","https://openalex.org/W2098865355","https://openalex.org/W2127849236","https://openalex.org/W2160473997","https://openalex.org/W2170931744"],"related_works":["https://openalex.org/W2012288173","https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W1968538666","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2097662580","https://openalex.org/W3199302685","https://openalex.org/W2344072770","https://openalex.org/W2389163612"],"abstract_inverted_index":{"We":[0,25,48],"describe":[1],"an":[2],"approach":[3],"to":[4,45],"crowdsource":[5],"the":[6],"evaluation":[7],"of":[8,32],"TTS":[9],"systems":[10],"by":[11],"preference":[12,63],"tests":[13],"and":[14,42],"report":[15],"on":[16],"lessons":[17],"learnt":[18],"from":[19],"running":[20],"127":[21],"real-life":[22],"crowdsourced":[23],"tests.":[24],"show":[26],"that":[27,50],"at":[28],"least":[29],"one":[30],"type":[31],"cheating":[33,66],"becomes":[34],"more":[35],"prevalent":[36],"over":[37],"time":[38],"if":[39],"left":[40],"unchecked":[41],"develop":[43],"metrics":[44],"exclude":[46],"cheaters.":[47],"demonstrate":[49],"their":[51],"exclusion":[52],"improves":[53],"test":[54],"outcomes.":[55],"Index":[56],"Terms:":[57],"TTS,":[58],"speech":[59],"synthesis,":[60],"listening":[61],"test,":[62,64],"crowdsourcing,":[65]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":7},{"year":2019,"cited_by_count":9},{"year":2018,"cited_by_count":7},{"year":2017,"cited_by_count":10},{"year":2016,"cited_by_count":14},{"year":2015,"cited_by_count":5},{"year":2014,"cited_by_count":9},{"year":2013,"cited_by_count":8},{"year":2012,"cited_by_count":7}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}