{"id":"https://openalex.org/W7168399792","doi":"https://doi.org/10.1145/3805712.3809556","title":"Do Simulated Users Need to Remember? Analyzing the Impact of Memory Models in Conversational Search Evaluation","display_name":"Do Simulated Users Need to Remember? Analyzing the Impact of Memory Models in Conversational Search Evaluation","publication_year":2026,"publication_date":"2026-07-15","ids":{"openalex":"https://openalex.org/W7168399792","doi":"https://doi.org/10.1145/3805712.3809556"},"language":null,"primary_location":{"id":"doi:10.1145/3805712.3809556","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809556","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805712.3809556","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140808394","display_name":"Nailia Mirzakhmedova","orcid":"https://orcid.org/0000-0002-8143-1405"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Nailia Mirzakhmedova","raw_affiliation_strings":["Bauhaus Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":"https://orcid.org/0000-0002-8143-1405","affiliations":[{"raw_affiliation_string":"Bauhaus Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140924691","display_name":"Marcel Gohsen","orcid":"https://orcid.org/0000-0002-1020-6745"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Marcel Gohsen","raw_affiliation_strings":["Bauhaus Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":"https://orcid.org/0000-0002-1020-6745","affiliations":[{"raw_affiliation_string":"Bauhaus Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140926315","display_name":"Johannes Kiesel","orcid":"https://orcid.org/0000-0002-1617-6508"},"institutions":[{"id":"https://openalex.org/I4210101898","display_name":"GESIS - Leibniz Institute for the Social Sciences","ror":"https://ror.org/018afyw53","country_code":"DE","type":"facility","lineage":["https://openalex.org/I315704651","https://openalex.org/I4210101898"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Johannes Kiesel","raw_affiliation_strings":["GESIS - Leibniz Institute for the Social Sciences, Cologne, Germany"],"raw_orcid":"https://orcid.org/0000-0002-1617-6508","affiliations":[{"raw_affiliation_string":"GESIS - Leibniz Institute for the Social Sciences, Cologne, Germany","institution_ids":["https://openalex.org/I4210101898"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140958031","display_name":"Matthias Hagen","orcid":"https://orcid.org/0000-0002-9733-2890"},"institutions":[{"id":"https://openalex.org/I76198965","display_name":"Friedrich Schiller University Jena","ror":"https://ror.org/05qpz1x62","country_code":"DE","type":"education","lineage":["https://openalex.org/I76198965"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias Hagen","raw_affiliation_strings":["Friedrich-Schiller Universit\u00e4t Jena, Jena, Germany"],"raw_orcid":"https://orcid.org/0000-0002-9733-2890","affiliations":[{"raw_affiliation_string":"Friedrich-Schiller Universit\u00e4t Jena, Jena, Germany","institution_ids":["https://openalex.org/I76198965"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5140876032","display_name":"Benno Stein","orcid":"https://orcid.org/0000-0001-9033-2217"},"institutions":[{"id":"https://openalex.org/I51441396","display_name":"Bauhaus-Universit\u00e4t Weimar","ror":"https://ror.org/033bb5z47","country_code":"DE","type":"education","lineage":["https://openalex.org/I51441396"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Benno Stein","raw_affiliation_strings":["Bauhaus Universit\u00e4t Weimar, Weimar, Germany"],"raw_orcid":"https://orcid.org/0000-0001-9033-2217","affiliations":[{"raw_affiliation_string":"Bauhaus Universit\u00e4t Weimar, Weimar, Germany","institution_ids":["https://openalex.org/I51441396"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1359","last_page":"1370"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.27639999985694885},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.25850000977516174},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.25279998779296875},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.24120000004768372}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6517999768257141},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3792000114917755},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.30309998989105225},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2791999876499176},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25850000977516174},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.25279998779296875},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.24120000004768372},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.23589999973773956},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.2198999971151352}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3805712.3809556","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809556","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3805712.3809556","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3809556","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W222747623","https://openalex.org/W1545435642","https://openalex.org/W1798038286","https://openalex.org/W1969519340","https://openalex.org/W2022995284","https://openalex.org/W2038550763","https://openalex.org/W2039847928","https://openalex.org/W2465191537","https://openalex.org/W2530607235","https://openalex.org/W2590822507","https://openalex.org/W2788926820","https://openalex.org/W2889742741","https://openalex.org/W2909551589","https://openalex.org/W3035734059","https://openalex.org/W3126966255","https://openalex.org/W3195417707","https://openalex.org/W4212975704","https://openalex.org/W4225815573","https://openalex.org/W4226536440","https://openalex.org/W4284691825","https://openalex.org/W4287822877","https://openalex.org/W4288113479","https://openalex.org/W4302322961","https://openalex.org/W4306317758","https://openalex.org/W4316837538","https://openalex.org/W4367365649","https://openalex.org/W4377238805","https://openalex.org/W4379258905","https://openalex.org/W4379474662","https://openalex.org/W4385565522","https://openalex.org/W4385642073","https://openalex.org/W4385688511","https://openalex.org/W4387074860","https://openalex.org/W4391420795","https://openalex.org/W4392471527","https://openalex.org/W4392846390","https://openalex.org/W4393147158","https://openalex.org/W4393335874","https://openalex.org/W4396786282","https://openalex.org/W4396945227","https://openalex.org/W4399631598","https://openalex.org/W4400163193","https://openalex.org/W4401042817","https://openalex.org/W4401306979","https://openalex.org/W4401330297","https://openalex.org/W4401381513","https://openalex.org/W4402527084","https://openalex.org/W4403797051","https://openalex.org/W4405165399","https://openalex.org/W4405643789","https://openalex.org/W4410636398","https://openalex.org/W4412396342","https://openalex.org/W4413243547","https://openalex.org/W4416037320","https://openalex.org/W6910688193","https://openalex.org/W7133483856","https://openalex.org/W7133538538"],"related_works":[],"abstract_inverted_index":{"Conversational":[0],"search":[1,78,97,120,164],"systems":[2],"are":[3,34,198,222,231,237],"typically":[4],"evaluated":[5,223],"using":[6],"a":[7,18,27,56,64],"fixed":[8],"reference":[9],"collection":[10],"of":[11,63,76,83,95,142,149,155,159,209],"conversations":[12,23,154],"or":[13],"through":[14],"user":[15,32,52,65],"studies":[16,33],"with":[17,68,161,227],"live":[19],"system.":[20],"However,":[21],"fixed-reference":[22],"can":[24,175,189],"cover":[25],"only":[26],"few":[28],"plausible":[29],"conversations,":[30,184],"and":[31,37,48,66,115,158,201,235],"costly,":[35],"time-consuming,":[36],"often":[38],"hard":[39],"to":[40,90,108,126],"reproduce.":[41],"A":[42],"promising":[43],"alternative":[44],"that":[45,169,196],"avoids":[46],"coverage":[47],"cost":[49],"issues":[50],"is":[51],"simulation,":[53],"in":[54,131],"which":[55,122],"computer":[57],"program":[58],"takes":[59],"on":[60],"the":[61,69,74,81,86,140,147,191],"role":[62],"interacts":[67],"system":[70,143,178,210],"under":[71],"evaluation.":[72],"But":[73],"complexity":[75],"human":[77,183,215],"behavior":[79],"raises":[80],"question":[82],"how":[84],"''realistic''":[85],"simulations":[87,213],"actually":[88,138],"need":[89,107],"be":[91],"for":[92],"reliable":[93],"evaluations":[94],"conversational":[96,119,163],"systems.":[98,165],"In":[99],"this":[100,137],"paper,":[101],"we":[102,152],"ask:":[103],"Do":[104],"simulated":[105,156],"users":[106,112,157],"remember?":[109],"While":[110],"real":[111],"may":[113],"learn":[114],"forget":[116],"information":[117],"during":[118],"sessions,":[121],"inspired":[123],"previous":[124],"research":[125],"also":[127,194],"model":[128],"memory":[129,150,172,188,203],"capabilities":[130],"simulations,":[132],"it":[133],"remains":[134],"unclear":[135],"whether":[136],"influences":[139],"results":[141,167],"evaluations.":[144],"To":[145],"investigate":[146],"impact":[148],"modeling,":[151],"analyze":[153],"humans":[160],"four":[162],"Our":[166,233],"suggest":[168],"incorporating":[170,186],"long-term":[171],"into":[173],"simulators":[174,197],"help":[176],"reproduce":[177],"effectiveness":[179],"rankings":[180],"obtained":[181],"from":[182],"whereas":[185],"short-term":[187],"diminish":[190],"reproduction.":[192],"We":[193],"find":[195],"generally":[199],"valid":[200],"reproducible---and":[202],"modeling":[204],"even":[205],"increases":[206],"run-to-run":[207],"reproducibility":[208],"rankings---but":[211],"overall,":[212],"approximate":[214],"evaluation":[216],"scores":[217],"better":[218],"when":[219,225],"''helpful''":[220],"assistants":[221,226],"than":[224],"deteriorated":[228],"response":[229],"quality":[230],"assessed.":[232],"code":[234],"data":[236],"available":[238],"at":[239],"https://github.com/webis-de/SIGIR-26.":[240]},"counts_by_year":[],"updated_date":"2026-07-17T05:52:16.776730","created_date":"2026-07-16T00:00:00"}