{"id":"https://openalex.org/W7133812015","doi":"https://doi.org/10.48550/arxiv.2603.03415","title":"Farther the Shift, Sparser the Representation: Analyzing OOD Mechanisms in LLMs","display_name":"Farther the Shift, Sparser the Representation: Analyzing OOD Mechanisms in LLMs","publication_year":2026,"publication_date":"2026-03-03","ids":{"openalex":"https://openalex.org/W7133812015","doi":"https://doi.org/10.48550/arxiv.2603.03415"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2603.03415","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128202802","display_name":"Mingyu Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Mingyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128189394","display_name":"Yutong Yin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Yutong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047674908","display_name":"Jingcheng Niu","orcid":"https://orcid.org/0000-0002-2358-9466"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Niu, Jingcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128203827","display_name":"Qingcheng Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Qingcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128217977","display_name":"Wujiang Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Wujiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114452506","display_name":"Mengnan Du","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Mengnan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128209436","display_name":"Wei Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128166093","display_name":"Zhaoran Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhaoran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128171714","display_name":"Tianlong Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Tianlong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128175913","display_name":"Dimitris N. Metaxas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Metaxas, Dimitris N.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.26914055,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3012999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3012999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.19539999961853027,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.05620000138878822,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5396000146865845},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5317999720573425},{"id":"https://openalex.org/keywords/mechanism","display_name":"Mechanism (biology)","score":0.44940000772476196},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.41029998660087585},{"id":"https://openalex.org/keywords/schedule","display_name":"Schedule","score":0.3977999985218048},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.3675000071525574},{"id":"https://openalex.org/keywords/causality","display_name":"Causality (physics)","score":0.36410000920295715},{"id":"https://openalex.org/keywords/sequence-learning","display_name":"Sequence learning","score":0.3637999892234802},{"id":"https://openalex.org/keywords/linear-subspace","display_name":"Linear subspace","score":0.33970001339912415}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5867000222206116},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5396000146865845},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5317999720573425},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5284000039100647},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.44940000772476196},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.41029998660087585},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.4072999954223633},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.40230000019073486},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.3977999985218048},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.3675000071525574},{"id":"https://openalex.org/C64357122","wikidata":"https://www.wikidata.org/wiki/Q1149766","display_name":"Causality (physics)","level":2,"score":0.36410000920295715},{"id":"https://openalex.org/C40506919","wikidata":"https://www.wikidata.org/wiki/Q7452469","display_name":"Sequence learning","level":2,"score":0.3637999892234802},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3528999984264374},{"id":"https://openalex.org/C12362212","wikidata":"https://www.wikidata.org/wiki/Q728435","display_name":"Linear subspace","level":2,"score":0.33970001339912415},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.3393000066280365},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3384000062942505},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3174999952316284},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.3068999946117401},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.30660000443458557},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2944999933242798},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.28450000286102295},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2842999994754791},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.27410000562667847},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2732999920845032},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C143724316","wikidata":"https://www.wikidata.org/wiki/Q312468","display_name":"Series (stratigraphy)","level":2,"score":0.26269999146461487},{"id":"https://openalex.org/C11671645","wikidata":"https://www.wikidata.org/wiki/Q5054567","display_name":"Causal model","level":2,"score":0.2619999945163727},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2603999972343445},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.25999999046325684},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.2540999948978424}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2603.03415","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2603.03415","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.03415","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2603.03415","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"score":0.8216608762741089,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0,59],"this":[1,114,129],"work,":[2],"we":[3,111,131],"investigate":[4],"how":[5,161],"Large":[6],"Language":[7],"Models":[8],"(LLMs)":[9],"adapt":[10],"their":[11],"internal":[12],"representations":[13],"when":[14],"encountering":[15],"inputs":[16,88],"of":[17,24,54,103],"increasing":[18],"difficulty,":[19],"quantified":[20],"as":[21,35],"the":[22,50,63,65,67,96,172],"degree":[23],"out-of-distribution":[25],"(OOD)":[26],"shift.":[27],"We":[28],"reveal":[29],"a":[30,101,107,138],"consistent":[31],"and":[32,77],"quantifiable":[33],"phenomenon:":[34],"task":[36],"difficulty":[37],"increases,":[38],"whether":[39],"through":[40],"harder":[41],"reasoning":[42,125],"questions,":[43],"longer":[44],"contexts,":[45],"or":[46,86],"adding":[47],"answer":[48],"choices,":[49],"last":[51,97],"hidden":[52,98],"states":[53],"LLMs":[55,162],"become":[56],"substantially":[57],"sparser.":[58],"short,":[60],"\\textbf{\\textit{the":[61],"farther":[62],"shift,":[64],"sparser":[66],"representations}}.":[68],"This":[69],"sparsity--difficulty":[70],"relation":[71],"is":[72,116,169],"observable":[73],"across":[74],"diverse":[75],"models":[76,82],"domains,":[78],"suggesting":[79],"that":[80,113,140],"language":[81],"respond":[83],"to":[84,145,150],"unfamiliar":[85],"complex":[87],"by":[89],"concentrating":[90],"computation":[91],"into":[92,160],"specialized":[93],"subspaces":[94],"in":[95],"state.":[99],"Through":[100],"series":[102],"controlled":[104],"analyses":[105],"with":[106],"learning":[108],"dynamic":[109],"explanation,":[110],"demonstrate":[112],"sparsity":[115,144],"not":[117],"incidental":[118],"but":[119],"an":[120],"adaptive":[121],"mechanism":[122],"for":[123],"stabilizing":[124],"under":[126],"OOD.":[127],"Leveraging":[128],"insight,":[130],"design":[132],"\\textit{Sparsity-Guided":[133],"Curriculum":[134],"In-Context":[135],"Learning":[136],"(SG-ICL)},":[137],"strategy":[139],"explicitly":[141],"uses":[142],"representation":[143],"schedule":[146],"few-shot":[147],"demonstrations,":[148],"leading":[149],"considerable":[151],"performance":[152],"enhancements.":[153],"Our":[154],"study":[155],"provides":[156],"new":[157],"mechanistic":[158],"insights":[159],"internalize":[163],"OOD":[164],"challenges.":[165],"The":[166],"source":[167],"code":[168],"available":[170],"at":[171],"URL:":[173],"https://github.com/MingyuJ666/sparsityLLM.":[174]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-03-06T00:00:00"}