{"id":"https://openalex.org/W2189217339","doi":"https://doi.org/10.1145/2736277.2741669","title":"Automatic Online Evaluation of Intelligent Assistants","display_name":"Automatic Online Evaluation of Intelligent Assistants","publication_year":2015,"publication_date":"2015-05-18","ids":{"openalex":"https://openalex.org/W2189217339","doi":"https://doi.org/10.1145/2736277.2741669","mag":"2189217339"},"language":"en","primary_location":{"id":"doi:10.1145/2736277.2741669","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2736277.2741669","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th International Conference on World Wide Web","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019675617","display_name":"Jiepu Jiang","orcid":"https://orcid.org/0000-0002-8207-7452"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jiepu Jiang","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, MA, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021000040","display_name":"Ahmed Hassan Awadallah","orcid":"https://orcid.org/0000-0001-6426-3537"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ahmed Hassan Awadallah","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000992993","display_name":"Rosie Jones","orcid":"https://orcid.org/0009-0000-3821-1207"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rosie Jones","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080908821","display_name":"Umut \u00d6zertem","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Umut Ozertem","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108365460","display_name":"Imed Zitouni","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Imed Zitouni","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021560352","display_name":"Ranjitha Gurunath Kulkarni","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ranjitha Gurunath Kulkarni","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108225108","display_name":"Omar Zia Khan","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Omar Zia Khan","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5019675617"],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":28.4926,"has_fulltext":false,"cited_by_count":149,"citation_normalized_percentile":{"value":0.99639269,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"506","last_page":"516"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8353462219238281},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.731111466884613},{"id":"https://openalex.org/keywords/dialog-box","display_name":"Dialog box","score":0.722736120223999},{"id":"https://openalex.org/keywords/session","display_name":"Session (web analytics)","score":0.6976624131202698},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.5238549709320068},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5117799639701843},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4607582688331604},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.41639935970306396},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.338525652885437},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2801951467990875}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8353462219238281},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.731111466884613},{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.722736120223999},{"id":"https://openalex.org/C2779182362","wikidata":"https://www.wikidata.org/wiki/Q17126187","display_name":"Session (web analytics)","level":2,"score":0.6976624131202698},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5238549709320068},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5117799639701843},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4607582688331604},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.41639935970306396},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.338525652885437},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2801951467990875},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/2736277.2741669","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2736277.2741669","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th International Conference on World Wide Web","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.695.5528","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.695.5528","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.www2015.it/documents/proceedings/proceedings/p506.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.696.2011","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.696.2011","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://maroo.cs.umass.edu/pub/web/getpdf.php?id%3D1177","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.721.7855","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.721.7855","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://people.cs.umass.edu/%7Ejpjiang/papers/www15_cortana_sat_final.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W157703060","https://openalex.org/W648947103","https://openalex.org/W1562973825","https://openalex.org/W1597379241","https://openalex.org/W1597441641","https://openalex.org/W1649407914","https://openalex.org/W1965237068","https://openalex.org/W1968101325","https://openalex.org/W1973120253","https://openalex.org/W1975244201","https://openalex.org/W1978616112","https://openalex.org/W2007750197","https://openalex.org/W2008251177","https://openalex.org/W2024046085","https://openalex.org/W2028825255","https://openalex.org/W2058473041","https://openalex.org/W2069870183","https://openalex.org/W2096946253","https://openalex.org/W2104255729","https://openalex.org/W2106576032","https://openalex.org/W2114418126","https://openalex.org/W2119049224","https://openalex.org/W2123937625","https://openalex.org/W2124063543","https://openalex.org/W2126891226","https://openalex.org/W2128970689","https://openalex.org/W2149590690","https://openalex.org/W2151814822","https://openalex.org/W2152693361","https://openalex.org/W2158254843","https://openalex.org/W2158312787","https://openalex.org/W2165474780","https://openalex.org/W2170443290","https://openalex.org/W2401425944","https://openalex.org/W4285719527","https://openalex.org/W6713409546","https://openalex.org/W6771752040"],"related_works":["https://openalex.org/W2098987383","https://openalex.org/W2795961259","https://openalex.org/W2117933979","https://openalex.org/W4298396513","https://openalex.org/W2283130723","https://openalex.org/W103938586","https://openalex.org/W2104718772","https://openalex.org/W2258651510","https://openalex.org/W1965155517","https://openalex.org/W2912415823"],"abstract_inverted_index":{"Voice-activated":[0],"intelligent":[1,85,101,128],"assistants,":[2],"such":[3],"as":[4,103,105],"Siri,":[5],"Google":[6],"Now,":[7],"and":[8,26,37,47,75,111,121,126,175,181,205,232],"Cortana,":[9],"are":[10,97],"prevalent":[11],"on":[12,200],"mobile":[13],"devices.":[14],"However,":[15],"it":[16,54],"is":[17,55,64,139,156],"challenging":[18],"to":[19,23,57,68,93,131,171,189],"evaluate":[20,58,80,120,214],"them":[21],"due":[22],"the":[24,65,100,132,154,165,176],"varied":[25],"evolving":[27],"number":[28],"of":[29,51,145,178,210],"tasks":[30,82,124],"supported,":[31],"e.g.,":[32,153],"voice":[33,211],"command,":[34],"web":[35,201],"search,":[36],"chat.":[38],"Since":[39],"each":[40,59],"task":[41,60],"may":[42],"have":[43],"its":[44,106],"own":[45],"procedure":[46],"a":[48,169,221],"unique":[49],"form":[50],"correct":[52],"answers,":[53],"expensive":[56],"individually.":[61],"This":[62],"paper":[63],"first":[66],"attempt":[67],"solve":[69],"this":[70,115],"challenge.":[71],"We":[72,87,163,184,213],"develop":[73],"consistent":[74],"automatic":[76,143],"approaches":[77],"that":[78],"can":[79,118,228],"different":[81,123],"in":[83,168],"voice-activated":[84],"assistants.":[86],"use":[88,164],"implicit":[89],"feedback":[90],"from":[91,197,220],"users":[92,96],"predict":[94,172],"whether":[95],"satisfied":[98],"with":[99],"assistant":[102],"well":[104],"components,":[107],"i.e.,":[108],"speech":[109,179],"recognition":[110,180],"intent":[112,182],"classification.":[113,183],"Using":[114],"approach,":[116,193],"we":[117],"potentially":[119],"compare":[122],"within":[125],"across":[127],"assistants":[129],"ac-cording":[130],"predicted":[133],"user":[134,155,173,222],"satisfaction":[135,174,203],"rates.":[136],"Our":[137],"approach":[138,216,227],"characterized":[140],"by":[141],"an":[142,161],"scheme":[144],"categorizing":[146],"user-system":[147],"interaction":[148],"into":[149],"task-independent":[150],"dialog":[151],"actions,":[152],"commanding,":[157],"selecting,":[158],"or":[159],"confirming":[160],"action.":[162],"action":[166],"sequence":[167],"session":[170],"quality":[177],"also":[185],"incorporate":[186],"other":[187],"features":[188,195],"further":[190],"improve":[191],"our":[192,215,226],"including":[194],"derived":[196],"previous":[198],"work":[199],"search":[202],"prediction,":[204],"those":[206],"utilizing":[207],"acoustic":[208],"characteristics":[209],"requests.":[212],"using":[217],"data":[218],"collected":[219],"study.":[223],"Results":[224],"show":[225],"accurately":[229],"identify":[230],"satisfactory":[231],"unsatisfactory":[233],"sessions.":[234]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":11},{"year":2022,"cited_by_count":17},{"year":2021,"cited_by_count":14},{"year":2020,"cited_by_count":18},{"year":2019,"cited_by_count":16},{"year":2018,"cited_by_count":28},{"year":2017,"cited_by_count":21},{"year":2016,"cited_by_count":14},{"year":2015,"cited_by_count":1}],"updated_date":"2026-05-29T09:21:14.243279","created_date":"2025-10-10T00:00:00"}