{"id":"https://openalex.org/W3193746305","doi":"https://doi.org/10.1145/3451160","title":"How Am I Doing?: Evaluating Conversational Search Systems Offline","display_name":"How Am I Doing?: Evaluating Conversational Search Systems Offline","publication_year":2021,"publication_date":"2021-08-17","ids":{"openalex":"https://openalex.org/W3193746305","doi":"https://doi.org/10.1145/3451160","mag":"3193746305"},"language":"en","primary_location":{"id":"doi:10.1145/3451160","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3451160","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://discovery.ucl.ac.uk/10125575/1/How_Am_I_Doing-Evaluating_Conversational_Search_Systems_Offline.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058519912","display_name":"Aldo Lipani","orcid":"https://orcid.org/0000-0002-3643-6493"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Aldo Lipani","raw_affiliation_strings":["University College London, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078123101","display_name":"Ben Carterette","orcid":"https://orcid.org/0000-0001-9538-047X"},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ben Carterette","raw_affiliation_strings":["Spotify, New York, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Spotify, New York, United States","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076265623","display_name":"Emine Y\u0131lmaz","orcid":"https://orcid.org/0000-0002-3434-8932"},"institutions":[{"id":"https://openalex.org/I4210123934","display_name":"Amazon (United Kingdom)","ror":"https://ror.org/02xey9634","country_code":"GB","type":"company","lineage":["https://openalex.org/I1311688040","https://openalex.org/I4210123934"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Emine Yilmaz","raw_affiliation_strings":["University College London & Amazon, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London & Amazon, London, United Kingdom","institution_ids":["https://openalex.org/I4210123934","https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.94,"has_fulltext":true,"cited_by_count":44,"citation_normalized_percentile":{"value":0.95922732,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"39","issue":"4","first_page":"1","last_page":"22"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.792710542678833},{"id":"https://openalex.org/keywords/novelty","display_name":"Novelty","score":0.70071941614151},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.6376497745513916},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6210225820541382},{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.5874012112617493},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5233035087585449},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.47559985518455505},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.402732253074646},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3668819069862366},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3346153497695923}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.792710542678833},{"id":"https://openalex.org/C2778738651","wikidata":"https://www.wikidata.org/wiki/Q16546687","display_name":"Novelty","level":2,"score":0.70071941614151},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.6376497745513916},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6210225820541382},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.5874012112617493},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5233035087585449},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.47559985518455505},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.402732253074646},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3668819069862366},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3346153497695923},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C27206212","wikidata":"https://www.wikidata.org/wiki/Q34178","display_name":"Theology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3451160","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3451160","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:10125575","is_oa":true,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10125575/","pdf_url":"https://discovery.ucl.ac.uk/10125575/1/How_Am_I_Doing-Evaluating_Conversational_Search_Systems_Offline.pdf","source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM Transactions on Information Systems , 39 (4) , Article 51. (2021)","raw_type":"Article"}],"best_oa_location":{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:10125575","is_oa":true,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10125575/","pdf_url":"https://discovery.ucl.ac.uk/10125575/1/How_Am_I_Doing-Evaluating_Conversational_Search_Systems_Offline.pdf","source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM Transactions on Information Systems , 39 (4) , Article 51. (2021)","raw_type":"Article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7900000214576721}],"awards":[{"id":"https://openalex.org/G7929798828","display_name":null,"funder_award_id":"EP/P024289/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3193746305.pdf","grobid_xml":"https://content.openalex.org/works/W3193746305.grobid-xml"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W1517870057","https://openalex.org/W1591706642","https://openalex.org/W1607341183","https://openalex.org/W1609424028","https://openalex.org/W1968927634","https://openalex.org/W1985476803","https://openalex.org/W1999444901","https://openalex.org/W2027382829","https://openalex.org/W2039133058","https://openalex.org/W2101105183","https://openalex.org/W2113640060","https://openalex.org/W2132314908","https://openalex.org/W2294145134","https://openalex.org/W2295570185","https://openalex.org/W2328886022","https://openalex.org/W2349436533","https://openalex.org/W2470681930","https://openalex.org/W2575238773","https://openalex.org/W2590822507","https://openalex.org/W2756731689","https://openalex.org/W2782940392","https://openalex.org/W2786860129","https://openalex.org/W2799104453","https://openalex.org/W2806532810","https://openalex.org/W2810821963","https://openalex.org/W2888302696","https://openalex.org/W2891826200","https://openalex.org/W2916904544","https://openalex.org/W2951534261","https://openalex.org/W2955157382","https://openalex.org/W2955792106","https://openalex.org/W2963748441","https://openalex.org/W2963903950","https://openalex.org/W2966087730","https://openalex.org/W2972156721","https://openalex.org/W2976801151","https://openalex.org/W2983435475","https://openalex.org/W2987293115","https://openalex.org/W3006129315","https://openalex.org/W3013372641","https://openalex.org/W3015691653","https://openalex.org/W3022041859","https://openalex.org/W3102132500","https://openalex.org/W4300410021"],"related_works":["https://openalex.org/W2368605798","https://openalex.org/W2518037665","https://openalex.org/W2348524959","https://openalex.org/W2477036161","https://openalex.org/W2368049389","https://openalex.org/W2384861574","https://openalex.org/W4294565801","https://openalex.org/W2170801710","https://openalex.org/W2952704802","https://openalex.org/W1967082116"],"abstract_inverted_index":{"As":[0,179],"conversational":[1,39,94,107,201],"agents":[2],"like":[3],"Siri":[4],"and":[5,10,17,59,129,155,159,161,175],"Alexa":[6],"gain":[7],"in":[8,35,68,157,177],"popularity":[9],"use,":[11],"conversation":[12],"is":[13,143,165,185],"becoming":[14],"a":[15,101,111,125,194],"more":[16,18,53,60],"important":[19,37],"mode":[20],"of":[21,49,71,76,106,148,200],"interaction":[22,127,135],"for":[23,81,103,113,197],"search.":[24,95,202],"Conversational":[25],"search":[26,40,82,158],"shares":[27],"some":[28,36],"features":[29],"with":[30,117],"traditional":[31,79],"search,":[32,108],"but":[33],"differs":[34],"respects:":[38],"systems":[41],"are":[42],"less":[43],"likely":[44,54,61],"to":[45,55,62,93,132,137,152,167,189],"return":[46],"ranked":[47],"lists":[48],"results":[50],"(a":[51],"SERP),":[52],"involve":[56],"iterated":[57],"interactions,":[58],"feature":[63],"longer,":[64],"well-formed":[65],"user":[66,126,134,163],"queries":[67],"the":[69,86,139,146,162,168,186],"form":[70],"natural":[72],"language":[73],"questions.":[74],"Because":[75],"these":[77,191],"differences,":[78],"methods":[80],"evaluation":[83,105,121,199],"(such":[84],"as":[85,181],"Cranfield":[87],"paradigm)":[88],"do":[89],"not":[90],"translate":[91],"easily":[92],"In":[96],"this":[97,184],"work,":[98],"we":[99,182],"propose":[100],"framework":[102,142,196],"offline":[104,198],"which":[109],"includes":[110],"methodology":[112],"creating":[114],"test":[115],"collections":[116],"relevance":[118],"judgments,":[119],"an":[120,130],"measure":[122],"based":[123,144],"on":[124,145],"model,":[128],"approach":[131],"collecting":[133],"data":[136],"train":[138],"model.":[140],"The":[141],"idea":[147],"\u201csubtopics\u201d,":[149],"often":[150],"used":[151,176],"model":[153,164,171],"novelty":[154],"diversity":[156],"recommendation,":[160],"similar":[166],"geometric":[169],"browsing":[170],"introduced":[172],"by":[173],"RBP":[174],"ERR.":[178],"far":[180],"know,":[183],"first":[187],"work":[188],"combine":[190],"ideas":[192],"into":[193],"comprehensive":[195]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":15},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":5}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}