{"id":"https://openalex.org/W2139448311","doi":"https://doi.org/10.1109/slt.2010.5700864","title":"Collection of user judgments on spoken dialog system with crowdsourcing","display_name":"Collection of user judgments on spoken dialog system with crowdsourcing","alternate_titles":[],"publication_year":2010,"publication_date":"2010-12-01","ids":{"openalex":"https://openalex.org/W2139448311","doi":"https://doi.org/10.1109/slt.2010.5700864","mag":"2139448311"},"language":"en","primary_location":{"id":"doi:10.1109/slt.2010.5700864","doi":"https://doi.org/10.1109/slt.2010.5700864","is_oa":false,"landing_page_url":"https://doi.org/10.1109/slt.2010.5700864","pdf_url":null,"source":{"id":"https://openalex.org/S4363605953","display_name":"IEEE Spoken Language Technology Workshop (SLT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2010 IEEE Spoken Language Technology Workshop","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W2139448311"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080769371","display_name":"Zhaojun Yang","orcid":"https://orcid.org/0000-0002-0164-1577","observed_orcids":["https://orcid.org/0000-0002-0164-1577","https://orcid.org/0000-0002-2591-7283"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhaojun Yang","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of System Engineering and Engineering Management, The Chinese University of Hong Kong, Shatin, N.T., Hong Kong#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of System Engineering and Engineering Management, The Chinese University of Hong Kong, Shatin, N.T., Hong Kong#TAB#","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020234885","display_name":"Baichuan Li","orcid":"https://orcid.org/0009-0007-1730-8457","observed_orcids":["https://orcid.org/0009-0007-1730-8457"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Baichuan Li","raw_affiliation_strings":["Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065716081","display_name":"Yi Zhu","orcid":"https://orcid.org/0000-0001-7115-2544","observed_orcids":["https://orcid.org/0000-0001-7115-2544"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yi Zhu","raw_affiliation_strings":["Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042251906","display_name":"Irwin King","orcid":"https://orcid.org/0000-0001-8106-6447","observed_orcids":["https://orcid.org/0000-0001-8106-6447"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Irwin King","raw_affiliation_strings":["Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, The Chinese University of Hong Kong, Shatin, N. T. Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109252500","display_name":"Gina Levow","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gina Levow","raw_affiliation_strings":["Department of Linguistics, University of Washington, Seattle, WA, USA","Department of Linguistics, University of Washington, Seattle, 98195, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Linguistics, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]},{"raw_affiliation_string":"Department of Linguistics, University of Washington, Seattle, 98195, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019458385","display_name":"Helen M. L. Meng","orcid":"https://orcid.org/0000-0002-4427-3532","observed_orcids":["https://orcid.org/0000-0002-4427-3532"]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Helen Meng","raw_affiliation_strings":["Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","Department of System Engineering and Engineering Management, The Chinese University of Hong Kong, Shatin, N.T., Hong Kong#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of System Engineering and Engineering Management, Chinese University of Hong Kong, Sha Tin, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]},{"raw_affiliation_string":"Department of System Engineering and Engineering Management, The Chinese University of Hong Kong, Shatin, N.T., Hong Kong#TAB#","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.8446,"has_fulltext":false,"cited_by_count":36,"citation_normalized_percentile":{"value":0.93080357,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":87,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"277","last_page":"282"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.6122999787330627,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.6122999787330627,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.35740000009536743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10470","display_name":"Usability and User Interface Design","score":0.0031999999191612005,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spoken-dialogue-systems","display_name":"spoken dialogue systems","score":0.9940000176429749},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.7770000100135803},{"id":"https://openalex.org/keywords/amazon-mechanical-turk","display_name":"Amazon Mechanical Turk","score":0.8709999918937683},{"id":"https://openalex.org/keywords/user-judgment","display_name":"user judgment","score":0.7250000238418579},{"id":"https://openalex.org/keywords/human-intelligence-tasks","display_name":"human intelligence tasks","score":0.8019999861717224},{"id":"https://openalex.org/keywords/employee-reliability","display_name":"worker reliability","score":0.7239999771118164},{"id":"https://openalex.org/keywords/interobserver-agreement","display_name":"interobserver agreement","score":0.875},{"id":"https://openalex.org/keywords/task-completion","display_name":"task completion","score":0.8389999866485596}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8844592571258545},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7676680684089661},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.706480860710144},{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.6612817049026489},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6471081376075745},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.6020293831825256},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5806980729103088},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5554006099700928},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.5208562016487122},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4857790768146515},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.48323068022727966},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4466714859008789},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3904585838317871},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.35197532176971436},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.19006457924842834},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0826958417892456},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/slt.2010.5700864","doi":"https://doi.org/10.1109/slt.2010.5700864","is_oa":false,"landing_page_url":"https://doi.org/10.1109/slt.2010.5700864","pdf_url":null,"source":{"id":"https://openalex.org/S4363605953","display_name":"IEEE Spoken Language Technology Workshop (SLT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2010 IEEE Spoken Language Technology Workshop","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321026","display_name":"Ministry of Earth Sciences","ror":"https://ror.org/013cf5k59"},{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320322942","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W110183184","https://openalex.org/W125520553","https://openalex.org/W130097250","https://openalex.org/W178897730","https://openalex.org/W625387107","https://openalex.org/W1975016129","https://openalex.org/W2117488952","https://openalex.org/W2127495981","https://openalex.org/W2151814822","https://openalex.org/W2164777277","https://openalex.org/W2950483141","https://openalex.org/W3116407438"],"related_works":["https://openalex.org/W7099140814","https://openalex.org/W2172501646","https://openalex.org/W2251075997","https://openalex.org/W2578461668","https://openalex.org/W2008498844","https://openalex.org/W2767652681","https://openalex.org/W2290707757","https://openalex.org/W4404570588","https://openalex.org/W2889424988","https://openalex.org/W2400310643"],"abstract_inverted_index":{"This":[0,21],"paper":[1],"presents":[2],"an":[3,149],"initial":[4],"attempt":[5],"at":[6,65],"the":[7,62,78,82,87,94,111,118,121,144,162,165,171,191,203,214],"use":[8],"of":[9,13,46,59,68,72,77,89,103,113,120,152,156,164,174,193,205],"crowd-sourcing":[10],"for":[11,56,126,182,190],"collection":[12,122,198],"user":[14,124],"judgments":[15,125,197],"on":[16,24,91,202,213],"spoken":[17],"dialog":[18],"systems":[19],"(SDSs).":[20],"is":[22],"implemented":[23],"Amazon":[25],"Mechanical":[26],"Turk":[27],"(MTurk),":[28],"where":[29],"a":[30,34,43,53,69,196],"Requester":[31],"can":[32],"design":[33,54],"human":[35],"intelligence":[36],"task":[37,157,185],"(HIT)":[38],"to":[39,85,109],"be":[40,210],"performed":[41],"by":[42,131],"large":[44,70],"number":[45,71],"Workers":[47,90],"efficiently":[48],"and":[49,81,148],"cost-effectively.":[50],"We":[51,139],"describe":[52],"methodology":[55],"two":[57],"types":[58],"HITs":[60],"-":[61],"first":[63],"targets":[64],"fast":[66],"rating":[67],"dialogs":[73,129,153],"regarding":[74],"some":[75],"dimensions":[76],"SDS's":[79],"performance":[80],"second":[83,172],"aims":[84],"assess":[86],"reliability":[88,163],"MTurk":[92,146,194],"through":[93],"variability":[95],"in":[96,136,154,184],"ratings":[97,114,147,167,183],"across":[98],"different":[99],"Workers.":[100],"A":[101],"set":[102],"approval":[104],"rules":[105],"are":[106,134],"also":[107,177],"designed":[108],"control":[110],"quality":[112],"from":[115,168],"MTurk.":[116,169],"At":[117],"end":[119],"work,":[123],"about":[127],"8,000":[128],"rated":[130],"around":[132],"700Workers":[133],"collected":[135,215],"45":[137],"days.":[138],"observe":[140,178],"reasonable":[141],"consistency":[142],"between":[143],"manual":[145],"automatic":[150],"categorization":[151],"terms":[155],"completion,":[158],"which":[159,187],"partially":[160],"verifies":[161],"approved":[166],"From":[170],"type":[173],"HITs,":[175],"we":[176],"moderate":[179],"inter-rater":[180],"agreement":[181],"completion":[186],"provides":[188],"support":[189],"utilization":[192],"as":[195],"platform.":[199],"Further":[200],"research":[201],"exploration":[204],"SDS":[206],"evaluation":[207],"models":[208],"could":[209],"developed":[211],"based":[212],"corpus.":[216]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2139448311","counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":7},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":4},{"year":2013,"cited_by_count":4},{"year":2012,"cited_by_count":3}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2016-06-24T00:00:00"}
