{"id":"https://openalex.org/W3005989146","doi":"https://doi.org/10.1007/s13222-020-00333-z","title":"Comparing Wizard of Oz & Observational Studies for Conversational IR Evaluation","display_name":"Comparing Wizard of Oz & Observational Studies for Conversational IR Evaluation","publication_year":2020,"publication_date":"2020-02-10","ids":{"openalex":"https://openalex.org/W3005989146","doi":"https://doi.org/10.1007/s13222-020-00333-z","mag":"3005989146"},"language":"en","primary_location":{"id":"doi:10.1007/s13222-020-00333-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13222-020-00333-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s13222-020-00333-z.pdf","source":{"id":"https://openalex.org/S73012565","display_name":"Datenbank-Spektrum","issn_l":"1610-1995","issn":["1610-1995","1618-2162"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Datenbank-Spektrum","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s13222-020-00333-z.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041921674","display_name":"David Elsweiler","orcid":"https://orcid.org/0000-0002-5791-0641"},"institutions":[{"id":"https://openalex.org/I60668342","display_name":"University of Regensburg","ror":"https://ror.org/01eezs655","country_code":"DE","type":"education","lineage":["https://openalex.org/I60668342"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"David Elsweiler","raw_affiliation_strings":["University of Regensburg, Universit\u00e4tsstra\u00dfe\u00a031, 93053, Regensburg, Germany"],"affiliations":[{"raw_affiliation_string":"University of Regensburg, Universit\u00e4tsstra\u00dfe\u00a031, 93053, Regensburg, Germany","institution_ids":["https://openalex.org/I60668342"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082629529","display_name":"Alexander Frummet","orcid":"https://orcid.org/0000-0002-5982-7104"},"institutions":[{"id":"https://openalex.org/I60668342","display_name":"University of Regensburg","ror":"https://ror.org/01eezs655","country_code":"DE","type":"education","lineage":["https://openalex.org/I60668342"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Alexander Frummet","raw_affiliation_strings":["University of Regensburg, Universit\u00e4tsstra\u00dfe\u00a031, 93053, Regensburg, Germany"],"affiliations":[{"raw_affiliation_string":"University of Regensburg, Universit\u00e4tsstra\u00dfe\u00a031, 93053, Regensburg, Germany","institution_ids":["https://openalex.org/I60668342"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041130563","display_name":"Morgan Harvey","orcid":"https://orcid.org/0000-0001-5504-2089"},"institutions":[{"id":"https://openalex.org/I32394136","display_name":"Northumbria University","ror":"https://ror.org/049e6bc10","country_code":"GB","type":"education","lineage":["https://openalex.org/I32394136"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Morgan Harvey","raw_affiliation_strings":["Northumbria University, NE1 8ST, Newcastle upon Tyne, UK"],"affiliations":[{"raw_affiliation_string":"Northumbria University, NE1 8ST, Newcastle upon Tyne, UK","institution_ids":["https://openalex.org/I32394136"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5041921674"],"corresponding_institution_ids":["https://openalex.org/I60668342"],"apc_list":{"value":2380,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2380,"currency":"EUR","value_usd":2890},"fwci":0.1353,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.52699188,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":"20","issue":"1","first_page":"37","last_page":"41"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/wizard-of-oz","display_name":"Wizard of oz","score":0.9147212505340576},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7656705379486084},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.6573605537414551},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6036297678947449},{"id":"https://openalex.org/keywords/dialog-system","display_name":"Dialog system","score":0.4809957444667816},{"id":"https://openalex.org/keywords/observational-study","display_name":"Observational study","score":0.47643834352493286},{"id":"https://openalex.org/keywords/wizard","display_name":"Wizard","score":0.44463470578193665},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.415406197309494},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.393847793340683},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.37960004806518555},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2946970462799072},{"id":"https://openalex.org/keywords/dialog-box","display_name":"Dialog box","score":0.13158580660820007}],"concepts":[{"id":"https://openalex.org/C2987423310","wikidata":"https://www.wikidata.org/wiki/Q18159725","display_name":"Wizard of oz","level":2,"score":0.9147212505340576},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7656705379486084},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6573605537414551},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6036297678947449},{"id":"https://openalex.org/C190954187","wikidata":"https://www.wikidata.org/wiki/Q5270587","display_name":"Dialog system","level":3,"score":0.4809957444667816},{"id":"https://openalex.org/C23131810","wikidata":"https://www.wikidata.org/wiki/Q818574","display_name":"Observational study","level":2,"score":0.47643834352493286},{"id":"https://openalex.org/C2776784076","wikidata":"https://www.wikidata.org/wiki/Q580334","display_name":"Wizard","level":2,"score":0.44463470578193665},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.415406197309494},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.393847793340683},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37960004806518555},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2946970462799072},{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.13158580660820007},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1007/s13222-020-00333-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13222-020-00333-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s13222-020-00333-z.pdf","source":{"id":"https://openalex.org/S73012565","display_name":"Datenbank-Spektrum","issn_l":"1610-1995","issn":["1610-1995","1618-2162"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Datenbank-Spektrum","raw_type":"journal-article"},{"id":"pmh:oai:epub.uni-regensburg.de:43374","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401229","display_name":"University of Regensburg Publication Server (University of Regensburg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I60668342","host_organization_name":"University of Regensburg","host_organization_lineage":["https://openalex.org/I60668342"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.5283/epub.43374","is_oa":true,"landing_page_url":"https://doi.org/10.5283/epub.43374","pdf_url":null,"source":{"id":"https://openalex.org/S4306401229","display_name":"University of Regensburg Publication Server (University of Regensburg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I60668342","host_organization_name":"University of Regensburg","host_organization_lineage":["https://openalex.org/I60668342"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.1007/s13222-020-00333-z","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13222-020-00333-z","pdf_url":"https://link.springer.com/content/pdf/10.1007/s13222-020-00333-z.pdf","source":{"id":"https://openalex.org/S73012565","display_name":"Datenbank-Spektrum","issn_l":"1610-1995","issn":["1610-1995","1618-2162"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Datenbank-Spektrum","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8399999737739563,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320323328","display_name":"Universit\u00e4t Regensburg","ror":"https://ror.org/01eezs655"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3005989146.pdf","grobid_xml":"https://content.openalex.org/works/W3005989146.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W2167431750","https://openalex.org/W2405187948","https://openalex.org/W2590822507","https://openalex.org/W2740543663","https://openalex.org/W2885631051","https://openalex.org/W2968287414","https://openalex.org/W2995859987","https://openalex.org/W3012288267","https://openalex.org/W3150923731"],"related_works":["https://openalex.org/W4246078203","https://openalex.org/W659328528","https://openalex.org/W1550648368","https://openalex.org/W2127135847","https://openalex.org/W4402443486","https://openalex.org/W603474643","https://openalex.org/W2152818631","https://openalex.org/W1545061903","https://openalex.org/W1504394863","https://openalex.org/W2757160751"],"abstract_inverted_index":{"Abstract":[0],"Systematic":[1],"and":[2,58,97,118],"repeatable":[3],"measurement":[4],"of":[5,18,124,143,153],"information":[6],"systems":[7,41],"via":[8,66],"test":[9],"collections,":[10],"the":[11,16,22,80,85,125,141,151],"Cranfield":[12],"model,":[13],"has":[14],"been":[15],"mainstay":[17],"Information":[19],"Retrieval":[20],"since":[21],"1960s.":[23],"However,":[24],"this":[25,70,147],"may":[26],"not":[27,49,134],"be":[28,62],"appropriate":[29],"for":[30,140],"newer,":[31],"more":[32],"interactive":[33],"systems,":[34],"such":[35],"as":[36],"Conversational":[37],"Search":[38],"agents.":[39,68,156],"Such":[40],"rely":[42],"on":[43],"Machine":[44],"Learning":[45],"technologies,":[46],"which":[47],"are":[48,113],"yet":[50],"sufficiently":[51],"advanced":[52],"to":[53,94,129],"permit":[54],"true":[55],"human-like":[56,120],"dialogues,":[57],"so":[59],"research":[60],"can":[61],"enabled":[63],"by":[64],"simulation":[65],"human":[67],"In":[69],"work":[71],"we":[72,132],"compare":[73],"dialogues":[74],"obtained":[75],"from":[76],"two":[77],"studies":[78,126,145],"with":[79,88,115],"same":[81],"context,":[82],"assistance":[83],"in":[84,122,146],"kitchen,":[86],"but":[87],"different":[89],"experimental":[90],"setups,":[91],"allowing":[92],"us":[93],"learn":[95],"about":[96],"evaluate":[98],"conversational":[99,155],"IR":[100],"systems.":[101],"We":[102],"discover":[103],"that":[104,119],"users":[105],"adapt":[106],"their":[107],"behaviour":[108],"when":[109],"they":[110,112],"think":[111],"interacting":[114],"a":[116],"system":[117],"conversations":[121],"one":[123],"were":[127],"unpredictable":[128],"an":[130],"extent":[131],"did":[133],"expect.":[135],"Our":[136],"results":[137],"have":[138],"implications":[139],"development":[142],"new":[144],"area":[148],"and,":[149],"ultimately,":[150],"design":[152],"future":[154]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}