{"id":"https://openalex.org/W7155053732","doi":"https://doi.org/10.48550/arxiv.2604.17650","title":"Measuring Distribution Shift in User Prompts and Its Effects on LLM Performance","display_name":"Measuring Distribution Shift in User Prompts and Its Effects on LLM Performance","publication_year":2026,"publication_date":"2026-04-19","ids":{"openalex":"https://openalex.org/W7155053732","doi":"https://doi.org/10.48550/arxiv.2604.17650"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.17650","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17650","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.17650","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134190472","display_name":"Parker Seegmiller","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Seegmiller, Parker","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5066969862","display_name":"Sarah Masud Preum","orcid":"https://orcid.org/0000-0002-7771-8323"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Preum, Sarah Masud","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.5296000242233276,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.5296000242233276,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.05139999836683273,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.04259999841451645,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/paradigm-shift","display_name":"Paradigm shift","score":0.5946000218391418},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.5600000023841858},{"id":"https://openalex.org/keywords/distribution","display_name":"Distribution (mathematics)","score":0.5378000140190125},{"id":"https://openalex.org/keywords/natural-experiment","display_name":"Natural experiment","score":0.4560999870300293},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.34150001406669617}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6097000241279602},{"id":"https://openalex.org/C43540301","wikidata":"https://www.wikidata.org/wiki/Q689971","display_name":"Paradigm shift","level":2,"score":0.5946000218391418},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.5600000023841858},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.5378000140190125},{"id":"https://openalex.org/C49630185","wikidata":"https://www.wikidata.org/wiki/Q6980675","display_name":"Natural experiment","level":2,"score":0.4560999870300293},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.36039999127388},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.34150001406669617},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.27639999985694885},{"id":"https://openalex.org/C104759252","wikidata":"https://www.wikidata.org/wiki/Q187120","display_name":"Time shifting","level":3,"score":0.26440000534057617},{"id":"https://openalex.org/C67712803","wikidata":"https://www.wikidata.org/wiki/Q7901853","display_name":"User modeling","level":3,"score":0.26269999146461487}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.17650","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17650","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.17650","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17650","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLMs":[0,78],"are":[1,25],"increasingly":[2],"deployed":[3,29,77,110,165],"in":[4,70,152,164],"dynamic,":[5],"real-world":[6,119],"settings,":[7],"where":[8],"the":[9,84,107,213],"distribution":[10,34,68,100,191],"of":[11,64,109,134],"user":[12,52,126,153,208,229],"prompts":[13],"can":[14],"shift":[15,35,69,101,122,192],"substantially":[16],"over":[17,124,193,204],"time":[18,205],"as":[19],"new":[20],"tasks,":[21],"prompts,":[22,140],"and":[23,73,102,128,141,179,185,206,227],"users":[24,174],"introduced":[26],"to":[27,40,56,219],"a":[28,37,93,114,132],"model.":[30],"Such":[31],"natural":[32,66,98,189],"prompt":[33,67,89,99,121,154,190],"poses":[36],"major":[38],"challenge":[39],"LLM":[41,85,199,221],"reliability,":[42],"particularly":[43,171],"for":[44,48,96,216],"specialized":[45],"models":[46,136,184],"designed":[47],"narrow":[49],"domains":[50],"or":[51],"populations.":[53,230],"Despite":[54],"attention":[55],"out-of-distribution":[57],"robustness,":[58],"there":[59],"is":[60,170,186],"very":[61],"limited":[62],"exploration":[63],"measuring":[65],"prior":[71],"work,":[72],"its":[74,104],"impact":[75],"on":[76,106,137,143],"remains":[79,223],"poorly":[80],"understood.":[81],"We":[82,112,146,195],"introduce":[83],"Evaluation":[86],"under":[87],"Natural":[88],"Shift":[90],"(LENS)":[91],"framework:":[92],"data-centric":[94],"approach":[95],"quantifying":[97],"evaluating":[103,142],"effect":[105],"performance":[108,159,168,222],"LLMs.":[111,166],"perform":[113],"large-scale":[115],"evaluation":[116],"using":[117],"192":[118],"post-deployment":[120],"settings":[123],"time,":[125],"group,":[127],"geographic":[129,180],"axes,":[130],"training":[131,139],"total":[133],"81":[135],"4.68M":[138],"57.6k":[144],"prompts.":[145],"find":[147],"that":[148],"even":[149],"moderate":[150],"shifts":[151],"behavior":[155],"correspond":[156],"with":[157,183,188],"large":[158],"drops":[160],"(73%":[161],"average":[162],"loss)":[163],"This":[167],"degradation":[169],"prevalent":[172],"when":[173],"from":[175],"different":[176],"latent":[177],"groups":[178],"regions":[181],"interact":[182],"correlated":[187],"time.":[194],"systematically":[196],"characterize":[197],"how":[198],"instruction":[200],"following":[201],"ability":[202],"degrades":[203],"between":[207],"groups.":[209],"Our":[210],"findings":[211],"highlight":[212],"critical":[214],"need":[215],"data-driven":[217],"monitoring":[218],"ensure":[220],"stable":[224],"across":[225],"diverse":[226],"evolving":[228]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-04-22T00:00:00"}