{"id":"https://openalex.org/W4415329381","doi":"https://doi.org/10.52202/085713-3940","title":"The Atlas of In-Context Learning: How Attention Heads Shape In-Context Retrieval Augmentation","display_name":"The Atlas of In-Context Learning: How Attention Heads Shape In-Context Retrieval Augmentation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4415329381","doi":"https://doi.org/10.52202/085713-3940"},"language":"en","primary_location":{"id":"doi:10.52202/085713-3940","doi":"https://doi.org/10.52202/085713-3940","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-3940","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.15807","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046230981","display_name":"Patrick Kahardipraja","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I2800274787","display_name":"Fraunhofer Institute for Telecommunications, Heinrich Hertz Institute","ror":"https://ror.org/02tbr6331","country_code":"DE","type":"facility","lineage":["https://openalex.org/I2800274787","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Patrick Kahardipraja","raw_affiliation_strings":["Fraunhofer Heinrich Hertz Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer Heinrich Hertz Institute","institution_ids":["https://openalex.org/I2800274787"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019269807","display_name":"Reduan Achtibat","orcid":"https://orcid.org/0009-0008-0052-3836","observed_orcids":["https://orcid.org/0009-0008-0052-3836"]},"institutions":[{"id":"https://openalex.org/I2800274787","display_name":"Fraunhofer Institute for Telecommunications, Heinrich Hertz Institute","ror":"https://ror.org/02tbr6331","country_code":"DE","type":"facility","lineage":["https://openalex.org/I2800274787","https://openalex.org/I4923324"]},{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Reduan Achtibat","raw_affiliation_strings":["Fraunhofer HHI & TU Berlin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer HHI & TU Berlin","institution_ids":["https://openalex.org/I2800274787","https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034084717","display_name":"Thomas Wiegand","orcid":"https://orcid.org/0000-0002-1121-2581","observed_orcids":["https://orcid.org/0000-0002-1121-2581"]},"institutions":[{"id":"https://openalex.org/I2800274787","display_name":"Fraunhofer Institute for Telecommunications, Heinrich Hertz Institute","ror":"https://ror.org/02tbr6331","country_code":"DE","type":"facility","lineage":["https://openalex.org/I2800274787","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Thomas Wiegand","raw_affiliation_strings":["Fraunhofer HHI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer HHI","institution_ids":["https://openalex.org/I2800274787"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026451495","display_name":"Wojciech Samek","orcid":"https://orcid.org/0000-0002-6283-3265","observed_orcids":["https://orcid.org/0000-0002-6283-3265"]},"institutions":[{"id":"https://openalex.org/I2800274787","display_name":"Fraunhofer Institute for Telecommunications, Heinrich Hertz Institute","ror":"https://ror.org/02tbr6331","country_code":"DE","type":"facility","lineage":["https://openalex.org/I2800274787","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Wojciech Samek","raw_affiliation_strings":["Fraunhofer Heinrich Hertz Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer Heinrich Hertz Institute","institution_ids":["https://openalex.org/I2800274787"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017608155","display_name":"Sebastian Lapuschkin","orcid":"https://orcid.org/0000-0002-0762-7258","observed_orcids":["https://orcid.org/0000-0002-0762-7258"]},"institutions":[{"id":"https://openalex.org/I2800274787","display_name":"Fraunhofer Institute for Telecommunications, Heinrich Hertz Institute","ror":"https://ror.org/02tbr6331","country_code":"DE","type":"facility","lineage":["https://openalex.org/I2800274787","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sebastian Lapuschkin","raw_affiliation_strings":["Fraunhofer HHI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer HHI","institution_ids":["https://openalex.org/I2800274787"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.416,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.95649349,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"130953","last_page":"130997"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8276000022888184,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8276000022888184,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.13289999961853027,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.010200000368058681,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.9660000205039978},{"id":"https://openalex.org/keywords/retrieval-augmented-generation","display_name":"retrieval-augmented generation","score":0.9660000205039978},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8169999718666077},{"id":"https://openalex.org/keywords/attention-heads","display_name":"attention heads","score":0.9399999976158142},{"id":"https://openalex.org/keywords/attribution-analysis","display_name":"attribution analysis","score":0.5189999938011169},{"id":"https://openalex.org/keywords/question-answering","display_name":"question answering","score":0.777999997138977},{"id":"https://openalex.org/keywords/knowledge-tracing","display_name":"knowledge tracing","score":0.5690000057220459},{"id":"https://openalex.org/keywords/transparency","display_name":"transparency","score":0.32100000977516174},{"id":"https://openalex.org/keywords/interpretability","display_name":"interpretability","score":0.5950000286102295},{"id":"https://openalex.org/keywords/relational-knowledge","display_name":"relational knowledge","score":0.5669999718666077}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7896999716758728},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.7289000153541565},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.7132999897003174},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5688999891281128},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5138000249862671},{"id":"https://openalex.org/C2776673561","wikidata":"https://www.wikidata.org/wiki/Q655357","display_name":"Atlas (anatomy)","level":2,"score":0.4577000141143799},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.45019999146461487},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4408000111579895},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.43209999799728394},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.39820000529289246},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.3422999978065491},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.3133000135421753},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.31279999017715454},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3001999855041504},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.29829999804496765},{"id":"https://openalex.org/C5655090","wikidata":"https://www.wikidata.org/wiki/Q192588","display_name":"Relational database","level":2,"score":0.2971000075340271},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.28299999237060547},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2784999907016754},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.26899999380111694},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.26739999651908875},{"id":"https://openalex.org/C24574437","wikidata":"https://www.wikidata.org/wiki/Q7135228","display_name":"Parametric model","level":3,"score":0.26669999957084656},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.2547999918460846},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.2538999915122986},{"id":"https://openalex.org/C74672266","wikidata":"https://www.wikidata.org/wiki/Q815859","display_name":"Language acquisition","level":2,"score":0.25209999084472656},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.52202/085713-3940","doi":"https://doi.org/10.52202/085713-3940","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-3940","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2505.15807","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.15807","pdf_url":"https://arxiv.org/pdf/2505.15807","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:publica.fraunhofer.de:publica/523628","doi":null,"is_oa":false,"landing_page_url":"https://publica.fraunhofer.de/handle/publica/523628","pdf_url":null,"source":{"id":"https://openalex.org/S4306400318","display_name":"Fraunhofer-Publica (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"},{"id":"doi:10.48550/arxiv.2505.15807","doi":"https://doi.org/10.48550/arxiv.2505.15807","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.15807","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.15807","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.15807","pdf_url":"https://arxiv.org/pdf/2505.15807","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415329381.pdf","grobid_xml":"https://content.openalex.org/works/W4415329381.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W1787224781","https://openalex.org/W2080133951","https://openalex.org/W2612431505","https://openalex.org/W2809324505","https://openalex.org/W2912924812","https://openalex.org/W2934842096","https://openalex.org/W2946794439","https://openalex.org/W2948140294","https://openalex.org/W2953337107","https://openalex.org/W2968210605","https://openalex.org/W2969624041","https://openalex.org/W2970476646","https://openalex.org/W2979826702","https://openalex.org/W2988421999","https://openalex.org/W2988937804","https://openalex.org/W3015575765","https://openalex.org/W3015883388","https://openalex.org/W3092292656","https://openalex.org/W3122241445","https://openalex.org/W3152884768","https://openalex.org/W3155807546","https://openalex.org/W3199011812","https://openalex.org/W4226484906","https://openalex.org/W4285175926","https://openalex.org/W4297412003","https://openalex.org/W4385571886","https://openalex.org/W4385572714","https://openalex.org/W4385574263","https://openalex.org/W4386897466","https://openalex.org/W4388778348","https://openalex.org/W4389518997","https://openalex.org/W4389520348","https://openalex.org/W4389520380","https://openalex.org/W4389520670","https://openalex.org/W4398757454","https://openalex.org/W4401306886","https://openalex.org/W4401306979","https://openalex.org/W4402670703","https://openalex.org/W4402671677","https://openalex.org/W4406060232","https://openalex.org/W4407806116","https://openalex.org/W4410356661"],"related_works":["https://openalex.org/W4402683814","https://openalex.org/W4395482887","https://openalex.org/W7221790700","https://openalex.org/W7150730858","https://openalex.org/W4414988062","https://openalex.org/W4312048837","https://openalex.org/W4415336014","https://openalex.org/W7061594915","https://openalex.org/W7148537405","https://openalex.org/W4412945085"],"abstract_inverted_index":{"Large":[0],"language":[1,128],"models":[2],"are":[3],"able":[4],"to":[5,9,57,95,111],"exploit":[6],"in-context":[7,36,63],"learning":[8],"access":[10],"external":[11],"knowledge":[12,116],"beyond":[13],"their":[14,84,92],"training":[15],"data":[16],"through":[17],"retrieval-augmentation.":[18],"While":[19],"promising,":[20],"its":[21],"inner":[22],"workings":[23],"remain":[24],"unclear.":[25],"In":[26],"this":[27],"work,":[28],"we":[29,86,106],"shed":[30],"light":[31],"on":[32],"the":[33,101,108,113,121],"mechanism":[34],"of":[35,49,115],"retrieval":[37],"augmentation":[38],"for":[39],"question":[40],"answering":[41],"by":[42],"viewing":[43],"a":[44,47],"prompt":[45],"as":[46],"composition":[48],"informational":[50],"components.":[51],"We":[52],"propose":[53],"an":[54],"attribution-based":[55],"method":[56],"identify":[58],"specialized":[59],"attention":[60,93],"heads,":[61],"revealing":[62],"heads":[64,75],"that":[65,76],"comprehend":[66],"instructions":[67],"and":[68,73,90,126],"retrieve":[69],"relevant":[70],"contextual":[71],"information,":[72],"parametric":[74],"store":[77],"entities'":[78],"relational":[79],"knowledge.":[80],"To":[81],"better":[82],"understand":[83],"roles,":[85],"extract":[87],"function":[88],"vectors":[89],"modify":[91],"weights":[94],"show":[96],"how":[97],"they":[98],"can":[99],"influence":[100],"answer":[102],"generation":[103],"process.":[104],"Finally,":[105],"leverage":[107],"gained":[109],"insights":[110],"trace":[112],"sources":[114],"used":[117],"during":[118],"inference,":[119],"paving":[120],"way":[122],"towards":[123],"more":[124],"safe":[125],"transparent":[127],"models.":[129]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":4}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-19T00:00:00"}