{"id":"https://openalex.org/W2139448311","doi":"https://doi.org/10.1109/slt.2010.5700864","title":"Collection of user judgments on spoken dialog system with crowdsourcing","display_name":"Collection of user judgments on spoken dialog system with crowdsourcing","publication_year":2010,"publication_date":"2010-12-01","ids":{"openalex":"https://openalex.org/W2139448311","doi":"https://doi.org/10.1109/slt.2010.5700864","mag":"2139448311"},"language":"en","primary_location":{"id":"doi:10.1109/slt.2010.5700864","is_oa":false,"landing_page_url":"https://doi.org/10.1109/slt.2010.5700864","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2010 IEEE Spoken Language Technology Workshop","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080769371","display_name":"Zhaojun Yang","orcid":"https://orcid.org/0000-0002-0164-1577"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zhaojun Yang","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of System Engineering and Engineering Management, The Chinese University of Hong Kong, Shatin, N.T., Hong Kong#TAB#"],"affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of System Engineering and Engineering Management, The Chinese University of Hong Kong, Shatin, N.T., Hong Kong#TAB#","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020234885","display_name":"Baichuan Li","orcid":"https://orcid.org/0009-0007-1730-8457"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Baichuan Li","raw_affiliation_strings":["Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065716081","display_name":"Yi Zhu","orcid":"https://orcid.org/0000-0001-7115-2544"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Zhu","raw_affiliation_strings":["Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042251906","display_name":"Irwin King","orcid":"https://orcid.org/0000-0001-8106-6447"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Irwin King","raw_affiliation_strings":["Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109252500","display_name":"Gina Levow","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gina Levow","raw_affiliation_strings":["Department of Linguistics, University of Washington, Seattle, WA, USA","Department of Linguistics, University of Washington, Seattle, 98195, USA"],"affiliations":[{"raw_affiliation_string":"Department of Linguistics, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]},{"raw_affiliation_string":"Department of Linguistics, University of Washington, Seattle, 98195, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019458385","display_name":"Helen Meng","orcid":"https://orcid.org/0000-0002-4427-3532"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Helen Meng","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of System Engineering and Engineering Management, The Chinese University of Hong Kong, Shatin, N.T., Hong Kong#TAB#"],"affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of System Engineering and Engineering Management, The Chinese University of Hong Kong, Shatin, N.T., Hong Kong#TAB#","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5080769371"],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":5.4126,"has_fulltext":false,"cited_by_count":34,"citation_normalized_percentile":{"value":0.95652678,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"277","last_page":"282"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8844592571258545},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7676680684089661},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.706480860710144},{"id":"https://openalex.org/keywords/dialog-box","display_name":"Dialog box","score":0.6612817049026489},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6471081376075745},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.6020293831825256},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5806980729103088},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5554006099700928},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.5208562016487122},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4857790768146515},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.48323068022727966},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4466714859008789},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3904585838317871},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.35197532176971436},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.19006457924842834},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.0826958417892456}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8844592571258545},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7676680684089661},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.706480860710144},{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.6612817049026489},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6471081376075745},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.6020293831825256},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5806980729103088},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5554006099700928},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.5208562016487122},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4857790768146515},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.48323068022727966},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4466714859008789},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3904585838317871},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.35197532176971436},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.19006457924842834},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0826958417892456},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/slt.2010.5700864","is_oa":false,"landing_page_url":"https://doi.org/10.1109/slt.2010.5700864","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2010 IEEE Spoken Language Technology Workshop","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321026","display_name":"Ministry of Earth Sciences","ror":"https://ror.org/013cf5k59"},{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320322942","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W110183184","https://openalex.org/W125520553","https://openalex.org/W130097250","https://openalex.org/W178897730","https://openalex.org/W625387107","https://openalex.org/W1975016129","https://openalex.org/W2117488952","https://openalex.org/W2127495981","https://openalex.org/W2151814822","https://openalex.org/W2164777277","https://openalex.org/W2950483141","https://openalex.org/W3116407438","https://openalex.org/W4211048027","https://openalex.org/W6604508155","https://openalex.org/W6763515994"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W1490650877"],"abstract_inverted_index":{"This":[0,21],"paper":[1],"presents":[2],"an":[3,149],"initial":[4],"attempt":[5],"at":[6,65],"the":[7,62,78,82,87,94,111,118,121,144,162,165,171,191,203,214],"use":[8],"of":[9,13,46,59,68,72,77,89,103,113,120,152,156,164,174,193,205],"crowd-sourcing":[10],"for":[11,56,126,182,190],"collection":[12,122,198],"user":[14,124],"judgments":[15,125,197],"on":[16,24,91,202,213],"spoken":[17],"dialog":[18],"systems":[19],"(SDSs).":[20],"is":[22],"implemented":[23],"Amazon":[25],"Mechanical":[26],"Turk":[27],"(MTurk),":[28],"where":[29],"a":[30,34,43,53,69,196],"Requester":[31],"can":[32],"design":[33,54],"human":[35],"intelligence":[36],"task":[37,157,185],"(HIT)":[38],"to":[39,85,109],"be":[40,210],"performed":[41],"by":[42,131],"large":[44,70],"number":[45,71],"Workers":[47,90],"efficiently":[48],"and":[49,81,148],"cost-effectively.":[50],"We":[51,139],"describe":[52],"methodology":[55],"two":[57],"types":[58],"HITs":[60],"-":[61],"first":[63],"targets":[64],"fast":[66],"rating":[67],"dialogs":[73,129,153],"regarding":[74],"some":[75],"dimensions":[76],"SDS's":[79],"performance":[80],"second":[83,172],"aims":[84],"assess":[86],"reliability":[88,163],"MTurk":[92,146,194],"through":[93],"variability":[95],"in":[96,136,154,184],"ratings":[97,114,147,167,183],"across":[98],"different":[99],"Workers.":[100],"A":[101],"set":[102],"approval":[104],"rules":[105],"are":[106,134],"also":[107,177],"designed":[108],"control":[110],"quality":[112],"from":[115,168],"MTurk.":[116,169],"At":[117],"end":[119],"work,":[123],"about":[127],"8,000":[128],"rated":[130],"around":[132],"700Workers":[133],"collected":[135,215],"45":[137],"days.":[138],"observe":[140,178],"reasonable":[141],"consistency":[142],"between":[143],"manual":[145],"automatic":[150],"categorization":[151],"terms":[155],"completion,":[158],"which":[159,187],"partially":[160],"verifies":[161],"approved":[166],"From":[170],"type":[173],"HITs,":[175],"we":[176],"moderate":[179],"inter-rater":[180],"agreement":[181],"completion":[186],"provides":[188],"support":[189],"utilization":[192],"as":[195],"platform.":[199],"Further":[200],"research":[201],"exploration":[204],"SDS":[206],"evaluation":[207],"models":[208],"could":[209],"developed":[211],"based":[212],"corpus.":[216]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":6},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":4},{"year":2012,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}