{"id":"https://openalex.org/W2146311236","doi":"https://doi.org/10.1017/s1351324908004932","title":"Questionnaires for eliciting evaluation data from users of interactive question answering systems","display_name":"Questionnaires for eliciting evaluation data from users of interactive question answering systems","publication_year":2008,"publication_date":"2008-11-06","ids":{"openalex":"https://openalex.org/W2146311236","doi":"https://doi.org/10.1017/s1351324908004932","mag":"2146311236"},"language":"en","primary_location":{"id":"doi:10.1017/s1351324908004932","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s1351324908004932","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069419532","display_name":"Diane Kelly","orcid":"https://orcid.org/0000-0003-2231-0192"},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"D. KELLY","raw_affiliation_strings":["University of North Carolina, Chapel Hill, NC 27599-3360, USA e-mail:","University of north carolina, chapel hill, nc 27599-3360, usa e-mail: dianek@email.unc.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of North Carolina, Chapel Hill, NC 27599-3360, USA e-mail:","institution_ids":["https://openalex.org/I114027177"]},{"raw_affiliation_string":"University of north carolina, chapel hill, nc 27599-3360, usa e-mail: dianek@email.unc.edu","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070503927","display_name":"Paul B. Kantor","orcid":"https://orcid.org/0000-0002-7985-8039"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"P. B. KANTOR","raw_affiliation_strings":["Rutgers University, New Brunswick, NJ 08901, USA e-mail:","Rutgers university, new brunswick, nj 08901, usa e-mail: kantor@scils.rutgers.edu#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, New Brunswick, NJ 08901, USA e-mail:","institution_ids":["https://openalex.org/I102322142"]},{"raw_affiliation_string":"Rutgers university, new brunswick, nj 08901, usa e-mail: kantor@scils.rutgers.edu#TAB#","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023244993","display_name":"Emile L. Morse","orcid":null},"institutions":[{"id":"https://openalex.org/I1321296531","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416","country_code":"US","type":"funder","lineage":["https://openalex.org/I1321296531","https://openalex.org/I1343035065"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"E. L. MORSE","raw_affiliation_strings":["National Institute of Standards & Technology, Gaithersburg, MD 20899, USA e-mail:","National institute of standards & technology, gaithersburg, md 20899, usa e-mail: emile.morse@nist.gov#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Standards & Technology, Gaithersburg, MD 20899, USA e-mail:","institution_ids":["https://openalex.org/I1321296531"]},{"raw_affiliation_string":"National institute of standards & technology, gaithersburg, md 20899, usa e-mail: emile.morse@nist.gov#TAB#","institution_ids":["https://openalex.org/I1321296531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059655101","display_name":"Jean Scholtz","orcid":null},"institutions":[{"id":"https://openalex.org/I142606810","display_name":"Pacific Northwest National Laboratory","ror":"https://ror.org/05h992307","country_code":"US","type":"facility","lineage":["https://openalex.org/I1325736334","https://openalex.org/I1330989302","https://openalex.org/I142606810","https://openalex.org/I39565521"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"J. SCHOLTZ","raw_affiliation_strings":["Pacific Northwest National Laboratory, Richland, WA 99352, USA e-mail:","Pacific northwest national laboratory, richland, wa 99352, usa e-mail: jean.scholtz@pnl.gov#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pacific Northwest National Laboratory, Richland, WA 99352, USA e-mail:","institution_ids":["https://openalex.org/I142606810"]},{"raw_affiliation_string":"Pacific northwest national laboratory, richland, wa 99352, usa e-mail: jean.scholtz@pnl.gov#TAB#","institution_ids":["https://openalex.org/I142606810"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5090118631","display_name":"Ying Sun","orcid":"https://orcid.org/0000-0001-6703-4270"},"institutions":[{"id":"https://openalex.org/I1327163397","display_name":"State University of New York","ror":"https://ror.org/01q1z8k08","country_code":"US","type":"education","lineage":["https://openalex.org/I1327163397"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Y. SUN","raw_affiliation_strings":["University at Buffalo, The State University of New York, Buffalo, NY 14260, USA e-mail:","University at buffalo, the state university of new york, buffalo, ny 14260, usa e-mail: sun3@buffalo.edu#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University at Buffalo, The State University of New York, Buffalo, NY 14260, USA e-mail:","institution_ids":["https://openalex.org/I63190737"]},{"raw_affiliation_string":"University at buffalo, the state university of new york, buffalo, ny 14260, usa e-mail: sun3@buffalo.edu#TAB#","institution_ids":["https://openalex.org/I1327163397"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5069419532"],"corresponding_institution_ids":["https://openalex.org/I114027177"],"apc_list":null,"apc_paid":null,"fwci":1.7623,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.8838777,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"15","issue":"1","first_page":"119","last_page":"141"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8446739912033081},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7475141286849976},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.7308147549629211},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6995669603347778},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.614656925201416},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5870646238327026},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5191613435745239},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.45698100328445435},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3900548815727234},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.34935736656188965}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8446739912033081},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7475141286849976},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.7308147549629211},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6995669603347778},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.614656925201416},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5870646238327026},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5191613435745239},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.45698100328445435},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3900548815727234},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.34935736656188965},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1017/s1351324908004932","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s1351324908004932","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":37,"referenced_works":["https://openalex.org/W12229740","https://openalex.org/W82090637","https://openalex.org/W91742397","https://openalex.org/W135190683","https://openalex.org/W177082382","https://openalex.org/W208548429","https://openalex.org/W392180792","https://openalex.org/W1523019309","https://openalex.org/W1547816677","https://openalex.org/W1556280220","https://openalex.org/W1560729591","https://openalex.org/W1594180095","https://openalex.org/W1729578042","https://openalex.org/W1967359578","https://openalex.org/W1975730959","https://openalex.org/W1980902933","https://openalex.org/W1999754132","https://openalex.org/W2024201830","https://openalex.org/W2037057633","https://openalex.org/W2037140704","https://openalex.org/W2037933327","https://openalex.org/W2048987619","https://openalex.org/W2054756825","https://openalex.org/W2091814450","https://openalex.org/W2151814822","https://openalex.org/W2153173351","https://openalex.org/W2156644463","https://openalex.org/W2157289187","https://openalex.org/W2167439408","https://openalex.org/W2322584079","https://openalex.org/W2592119231","https://openalex.org/W4239329905","https://openalex.org/W4242835034","https://openalex.org/W4244642959","https://openalex.org/W4285719527","https://openalex.org/W6608599795","https://openalex.org/W6633724138"],"related_works":["https://openalex.org/W986318368","https://openalex.org/W2000785801","https://openalex.org/W2384605597","https://openalex.org/W2384410913","https://openalex.org/W2352878646","https://openalex.org/W2990194547","https://openalex.org/W2004734601","https://openalex.org/W2130149817","https://openalex.org/W1480123525","https://openalex.org/W2620865396"],"abstract_inverted_index":{"Abstract":[0],"Evaluating":[1],"interactive":[2,50,73],"question":[3],"answering":[4],"(QA)":[5],"systems":[6],"with":[7,80,105],"real":[8],"users":[9],"can":[10,31],"be":[11,32],"challenging":[12],"because":[13],"traditional":[14],"evaluation":[15],"measures":[16],"based":[17],"on":[18],"the":[19,53,106,110,119,124],"relevance":[20,29],"of":[21,49,55,123],"items":[22],"returned":[23],"are":[24,99],"difficult":[25],"to":[26,70],"employ":[27],"since":[28],"judgments":[30],"unstable":[33],"in":[34,40,44],"multi-user":[35],"evaluations.":[36],"The":[37],"work":[38],"reported":[39],"this":[41],"paper":[42],"evaluates,":[43],"distinguishing":[45,102],"among":[46,103],"a":[47,58,71,89],"set":[48],"QA":[51,74],"systems,":[52,82,104],"effectiveness":[54],"three":[56,97],"questionnaires:":[57],"Cognitive":[59],"Workload":[60],"Questionnaire":[61,108],"(NASA":[62],"TLX),":[63],"and":[64,66,85,121],"Task":[65,107],"System":[67],"Questionnaires":[68,77,98],"customized":[69],"specific":[72],"application.":[75],"These":[76],"were":[78],"evaluated":[79],"four":[81],"seven":[83],"analysts,":[84],"eight":[86],"scenarios":[87],"during":[88],"2-week":[90],"workshop.":[91],"Overall,":[92],"results":[93],"demonstrate":[94],"that":[95],"all":[96],"effective":[100],"at":[101],"being":[109],"most":[111],"sensitive.":[112],"Results":[113],"also":[114],"provide":[115],"initial":[116],"support":[117],"for":[118],"validity":[120],"reliability":[122],"Questionnaires.":[125]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2012,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}