{"id":"https://openalex.org/W7108327977","doi":"https://doi.org/10.1145/3767695.3769502","title":"Investigating LLM Variability in Personalized Conversational Information Retrieval","display_name":"Investigating LLM Variability in Personalized Conversational Information Retrieval","publication_year":2025,"publication_date":"2025-12-03","ids":{"openalex":"https://openalex.org/W7108327977","doi":"https://doi.org/10.1145/3767695.3769502"},"language":"en","primary_location":{"id":"doi:10.1145/3767695.3769502","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3767695.3769502","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3767695.3769502","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Simon Lupart","orcid":"https://orcid.org/0009-0008-2383-4557"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Simon Lupart","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Dani\u00ebl van Dijk","orcid":"https://orcid.org/0009-0004-2571-842X"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Dani\u00ebl van Dijk","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Eric Langezaal","orcid":"https://orcid.org/0009-0008-5614-1749"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Eric Langezaal","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ian van Dort","orcid":"https://orcid.org/0009-0008-5881-760X"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Ian van Dort","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":null,"display_name":"Mohammad Aliannejadi","orcid":"https://orcid.org/0000-0002-9447-4172"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Mohammad Aliannejadi","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":6.1,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.96790325,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"353","last_page":"363"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.5483999848365784,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.5483999848365784,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.12559999525547028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.06120000034570694,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/personalization","display_name":"Personalization","score":0.7347999811172485},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5446000099182129},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.49079999327659607},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.46459999680519104},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.43320000171661377},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4059999883174896}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7788000106811523},{"id":"https://openalex.org/C183003079","wikidata":"https://www.wikidata.org/wiki/Q1000371","display_name":"Personalization","level":2,"score":0.7347999811172485},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5446000099182129},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5349000096321106},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.49079999327659607},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.46459999680519104},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.43320000171661377},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4059999883174896},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.3824000060558319},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3208000063896179},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.3188999891281128},{"id":"https://openalex.org/C99016210","wikidata":"https://www.wikidata.org/wiki/Q5488129","display_name":"Query expansion","level":2,"score":0.29739999771118164},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.272599995136261},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.26899999380111694},{"id":"https://openalex.org/C67712803","wikidata":"https://www.wikidata.org/wiki/Q7901853","display_name":"User modeling","level":3,"score":0.2542000114917755}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3767695.3769502","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3767695.3769502","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},{"id":"pmh:oai:dare.uva.nl:openaire/e393a2d8-bfce-425d-bda6-867de0f7c15e","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/investigating-llm-variability-in-personalized-conversational-information-retrieval(e393a2d8-bfce-425d-bda6-867de0f7c15e).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lupart, S, van Dijk, D, Langezaal, E, van Dort, I & Aliannejadi, M 2025, Investigating LLM Variability in Personalized Conversational Information Retrieval. in SIGIR-AP 2025 : Proceedings of the 2025 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region : December 7-10, 2025, Xi'an, China. Association for Computing Machinery, New York, NY, pp. 353-363, 3rd International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region, SIGIR-AP 2025, Xi'an, China, 7/12/25. https://doi.org/10.1145/3767695.3769502","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1145/3767695.3769502","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3767695.3769502","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5888522863388062,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W2590822507","https://openalex.org/W2810150893","https://openalex.org/W2970996870","https://openalex.org/W2998702515","https://openalex.org/W3034654340","https://openalex.org/W3044364015","https://openalex.org/W3044704040","https://openalex.org/W3154898636","https://openalex.org/W3180230246","https://openalex.org/W4284685693","https://openalex.org/W4284691825","https://openalex.org/W4384388099","https://openalex.org/W4389518951","https://openalex.org/W4389519413","https://openalex.org/W4389519885","https://openalex.org/W4393034763","https://openalex.org/W4396786282","https://openalex.org/W4401071793","https://openalex.org/W4402860127","https://openalex.org/W4403582548","https://openalex.org/W4404783104","https://openalex.org/W4412376958","https://openalex.org/W4412377802","https://openalex.org/W4412394859","https://openalex.org/W4412888106","https://openalex.org/W4412888450"],"related_works":[],"abstract_inverted_index":{"Personalized":[0,21],"Conversational":[1],"Information":[2],"Retrieval":[3],"(CIR)":[4],"has":[5],"seen":[6],"rapid":[7],"progress":[8],"in":[9,240,261],"recent":[10],"years,":[11],"driven":[12],"by":[13,28],"the":[14,52,60,114,149,153,217,241,247,272],"development":[15],"of":[16,62,165,219,274],"Large":[17],"Language":[18],"Models":[19],"(LLMs).":[20],"CIR":[22,65,258],"aims":[23],"to":[24,38,41,58,152,170,284],"enhance":[25,187],"document":[26],"retrieval":[27,188,265],"leveraging":[29],"user-specific":[30],"information,":[31],"such":[32],"as":[33],"preferences,":[34],"knowledge,":[35],"or":[36,266],"constraints,":[37],"tailor":[39],"responses":[40],"individual":[42],"needs.":[43],"A":[44],"key":[45],"resource":[46],"developed":[47],"for":[48,77,235,249,290],"this":[49,69,127],"task":[50],"is":[51],"TREC":[53,156],"iKAT":[54,157,212],"2023":[55],"dataset,":[56,159],"designed":[57],"evaluate":[59,161],"integration":[61],"personalization":[63,92],"into":[64,84],"pipelines.":[66],"Building":[67],"on":[68,110,140,211,214],"resource,":[70],"Mo":[71],"et":[72],"al.":[73],"explored":[74],"several":[75],"strategies":[76],"incorporating":[78],"Personal":[79],"Textual":[80],"Knowledge":[81],"Bases":[82],"(PTKB)":[83],"LLM-based":[85,191,257],"query":[86],"reformulation.":[87],"Their":[88],"findings":[89],"suggested":[90],"that":[91,99,183],"from":[93],"PTKB":[94],"could":[95],"be":[96],"detrimental":[97],"and":[98,124,133,144,160,173,178,206,252,263,279,287],"human":[100],"annotations":[101],"were":[102,108],"often":[103],"noisy.":[104],"However,":[105],"these":[106],"conclusions":[107],"based":[109],"single-run":[111],"experiments":[112],"using":[113],"commercial":[115],"GPT-3.5":[116,177],"Turbo":[117],"model,":[118],"raising":[119],"concerns":[120],"about":[121],"output":[122,142],"variability":[123,143,210],"repeatability.":[125],"In":[126],"reproducibility":[128],"study,":[129],"we":[130,245],"rigorously":[131],"reproduce":[132],"extend":[134],"their":[135],"work,":[136],"with":[137],"a":[138,162,232],"focus":[139],"LLM":[141],"model":[145],"generalization.":[146],"We":[147,200],"apply":[148],"original":[150,242],"methods":[151,193],"newly":[154],"released":[155],"2024":[158],"diverse":[163],"range":[164],"models,":[166,277],"including":[167],"Llama":[168],"(1B":[169],"70B),":[171],"Qwen-7B,":[172],"closed-source":[174],"models":[175],"like":[176],"GPT-4o-mini.":[179],"Our":[180],"results":[181],"show":[182],"human-selected":[184],"PTKBs":[185],"consistently":[186],"performance,":[189],"while":[190],"selection":[192],"do":[194],"not":[195,238],"reliably":[196],"outperform":[197],"manual":[198],"choices.":[199],"further":[201],"compare":[202],"variance":[203,228,253],"across":[204,276],"datasets":[205],"observe":[207],"substantially":[208],"higher":[209],"than":[213,229],"CAsT,":[215],"highlighting":[216],"challenges":[218],"evaluating":[220],"personalized":[221,291],"CIR.":[222,292],"Notably,":[223],"recall-oriented":[224],"metrics":[225],"exhibit":[226],"lower":[227],"precision-oriented":[230],"ones,":[231],"critical":[233],"insight":[234],"first-stage":[236],"retrievers,":[237],"addressed":[239],"study.":[243],"Finally,":[244],"underscore":[246],"need":[248],"multi-run":[250],"evaluations":[251],"reporting":[254],"when":[255],"assessing":[256],"systems,":[259],"especially":[260],"dense":[262],"sparse":[264],"in-context":[267],"learning":[268],"settings.":[269],"By":[270],"broadening":[271],"scope":[273],"evaluation":[275],"datasets,":[278],"metrics,":[280],"our":[281],"study":[282],"contributes":[283],"more":[285],"robust":[286],"generalizable":[288],"practices":[289]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-12-03T00:00:00"}