{"id":"https://openalex.org/W4417069110","doi":"https://doi.org/10.48550/arxiv.2503.23306","title":"Focus Directions Make Your Language Models Pay More Attention to Relevant Contexts","display_name":"Focus Directions Make Your Language Models Pay More Attention to Relevant Contexts","publication_year":2025,"publication_date":"2025-03-30","ids":{"openalex":"https://openalex.org/W4417069110","doi":"https://doi.org/10.48550/arxiv.2503.23306"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2503.23306","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.23306","pdf_url":"https://arxiv.org/pdf/2503.23306","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2503.23306","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010382904","display_name":"Youxiang Zhu","orcid":"https://orcid.org/0009-0000-7294-1596","observed_orcids":["https://orcid.org/0009-0000-7294-1596"]},"institutions":[{"id":"https://openalex.org/I33434090","display_name":"University of Massachusetts Boston","ror":"https://ror.org/04ydmy275","country_code":"US","type":"education","lineage":["https://openalex.org/I33434090"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhu, Youxiang","raw_affiliation_strings":["University of Massachusetts Boston ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Boston ,","institution_ids":["https://openalex.org/I33434090"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Ruochen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Li, Ruochen","raw_affiliation_strings":["Technische Universit\u00e4t M \u00fcnchen ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t M \u00fcnchen ,","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090003719","display_name":"Danqing Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Danqing","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062859783","display_name":"Daniel Haehn","orcid":"https://orcid.org/0000-0001-9144-3461","observed_orcids":["https://orcid.org/0000-0001-9144-3461"]},"institutions":[{"id":"https://openalex.org/I33434090","display_name":"University of Massachusetts Boston","ror":"https://ror.org/04ydmy275","country_code":"US","type":"education","lineage":["https://openalex.org/I33434090"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haehn, Daniel","raw_affiliation_strings":["University of Massachusetts Boston ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Boston ,","institution_ids":["https://openalex.org/I33434090"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101655447","display_name":"Xiaohui Liang","orcid":"https://orcid.org/0000-0003-4064-2393","observed_orcids":["https://orcid.org/0000-0003-4064-2393","https://orcid.org/0000-0001-6351-2538"]},"institutions":[{"id":"https://openalex.org/I33434090","display_name":"University of Massachusetts Boston","ror":"https://ror.org/04ydmy275","country_code":"US","type":"education","lineage":["https://openalex.org/I33434090"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liang, Xiaohui","raw_affiliation_strings":["University of Massachusetts Boston ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts Boston ,","institution_ids":["https://openalex.org/I33434090"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9369999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9369999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.02410000003874302,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.021299999207258224,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/long-context-large-language-models","display_name":"long-context large language models","score":0.9210000038146973},{"id":"https://openalex.org/keywords/attention-heads","display_name":"attention heads","score":0.6930000185966492},{"id":"https://openalex.org/keywords/attention-allocation","display_name":"attention allocation","score":0.7059999704360962},{"id":"https://openalex.org/keywords/distraction","display_name":"distraction","score":0.4399999976158142},{"id":"https://openalex.org/keywords/task-alignment","display_name":"task alignment","score":0.777999997138977}],"concepts":[{"id":"https://openalex.org/C2776378700","wikidata":"https://www.wikidata.org/wiki/Q3030775","display_name":"Distraction","level":2,"score":0.7807999849319458},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.7434999942779541},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6118000149726868},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5971999764442444},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5382000207901001},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5081999897956848},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.46860000491142273},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4302999973297119},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.40939998626708984},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.36890000104904175},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3495999872684479},{"id":"https://openalex.org/C76188268","wikidata":"https://www.wikidata.org/wiki/Q1783165","display_name":"Context effect","level":3,"score":0.32170000672340393},{"id":"https://openalex.org/C2993807640","wikidata":"https://www.wikidata.org/wiki/Q103709453","display_name":"Attention network","level":2,"score":0.31679999828338623},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2955999970436096},{"id":"https://openalex.org/C56995899","wikidata":"https://www.wikidata.org/wiki/Q1126687","display_name":"Focus group","level":2,"score":0.2930999994277954},{"id":"https://openalex.org/C191160116","wikidata":"https://www.wikidata.org/wiki/Q3423709","display_name":"Regulatory focus theory","level":3,"score":0.28700000047683716},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.28200000524520874},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.2628999948501587},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C2986089797","wikidata":"https://www.wikidata.org/wiki/Q6501338","display_name":"Visual attention","level":3,"score":0.25029999017715454},{"id":"https://openalex.org/C71611378","wikidata":"https://www.wikidata.org/wiki/Q5165191","display_name":"Contextual design","level":3,"score":0.2500999867916107}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2503.23306","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.23306","pdf_url":"https://arxiv.org/pdf/2503.23306","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2503.23306","doi":"https://doi.org/10.48550/arxiv.2503.23306","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2503.23306","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2503.23306","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.23306","pdf_url":"https://arxiv.org/pdf/2503.23306","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4417069110.pdf","grobid_xml":"https://content.openalex.org/works/W4417069110.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W2612431505","https://openalex.org/W2947497897","https://openalex.org/W2951534261","https://openalex.org/W2952862139","https://openalex.org/W2971418622","https://openalex.org/W3012396850","https://openalex.org/W3083182073","https://openalex.org/W3103682594","https://openalex.org/W4297412003","https://openalex.org/W4385570777","https://openalex.org/W4387294617","https://openalex.org/W4387323904","https://openalex.org/W4388514540","https://openalex.org/W4393027562","https://openalex.org/W4393903977","https://openalex.org/W4393942918","https://openalex.org/W4394778349","https://openalex.org/W4395482887","https://openalex.org/W4399912398","https://openalex.org/W4401306886","https://openalex.org/W4402670703","https://openalex.org/W4402671235","https://openalex.org/W4402671334","https://openalex.org/W4402683814","https://openalex.org/W4403885223","https://openalex.org/W4404782813","https://openalex.org/W4404783080","https://openalex.org/W4405655184","https://openalex.org/W4407806116","https://openalex.org/W4411119282","https://openalex.org/W4415800641","https://openalex.org/W7125799258","https://openalex.org/W7196971478"],"related_works":["https://openalex.org/W4404400378","https://openalex.org/W4407800411","https://openalex.org/W4404781337","https://openalex.org/W4404781668","https://openalex.org/W4414875734","https://openalex.org/W4400023197","https://openalex.org/W7160940084","https://openalex.org/W6979225786","https://openalex.org/W7133203447","https://openalex.org/W4405418834"],"abstract_inverted_index":{"Long-context":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"are":[5],"prone":[6],"to":[7,53,57,67,88,92,121],"be":[8,62],"distracted":[9],"by":[10,64],"irrelevant":[11],"contexts.":[12,69],"The":[13],"reason":[14],"for":[15],"distraction":[16,47],"remains":[17],"poorly":[18],"understood.":[19],"In":[20],"this":[21],"paper,":[22],"we":[23,44],"first":[24],"identify":[25,72],"the":[26,37,41,77,105,123,128],"contextual":[27,50],"heads,":[28,84],"a":[29],"special":[30],"group":[31],"of":[32,40,82,107,127],"attention":[33,39,56,66,91],"heads":[34,51],"that":[35,46],"control":[36],"overall":[38],"LLM.":[42],"Then,":[43],"demonstrate":[45],"arises":[48],"when":[49],"fail":[52],"allocate":[54,89],"sufficient":[55],"relevant":[58,93],"contexts":[59,94],"and":[60,79,114],"can":[61],"mitigated":[63],"increasing":[65],"these":[68,83],"We":[70,102,131],"further":[71,137],"focus":[73,108,117],"directions,":[74],"located":[75],"at":[76],"key":[78],"query":[80],"activations":[81],"which":[85,98],"enable":[86],"them":[87],"more":[90],"without":[95],"explicitly":[96],"specifying":[97],"context":[99],"is":[100],"relevant.":[101],"comprehensively":[103],"evaluate":[104],"effect":[106],"direction":[109],"on":[110,139],"various":[111],"long-context":[112,129,140],"tasks":[113],"find":[115],"out":[116],"directions":[118],"could":[119,135],"help":[120],"mitigate":[122],"poor":[124],"task":[125],"alignment":[126],"LLMs.":[130],"believe":[132],"our":[133],"findings":[134],"promote":[136],"research":[138],"LLM":[141],"alignment.":[142]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":6}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2025-10-10T00:00:00"}