{"id":"https://openalex.org/W7222852263","doi":null,"title":"Can Capability Enhancement be Localized?","display_name":"Can Capability Enhancement be Localized?","publication_year":2026,"publication_date":"2026-10-15","ids":{"openalex":"https://openalex.org/W7222852263"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.21294","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.21294","pdf_url":"https://arxiv.org/pdf/2610.21294","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.21294","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100773449","display_name":"Anh Tuan Nguyen","orcid":"https://orcid.org/0000-0001-6977-9970","observed_orcids":["https://orcid.org/0000-0001-6977-9970"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nguyen, Anh Tuan","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016624892","display_name":"Anna Bair","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bair, Anna","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5158986590","display_name":"Zico Kolter","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kolter, Zico","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7329000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7329000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.11069999635219574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.07530000060796738,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/training-dynamics","display_name":"training dynamics","score":0.6579999923706055},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"gradient descent","score":0.5429999828338623},{"id":"https://openalex.org/keywords/transformer-based-language-models","display_name":"transformer language models","score":0.6669999957084656},{"id":"https://openalex.org/keywords/head-removal","display_name":"head removal","score":0.40799999237060547},{"id":"https://openalex.org/keywords/task-specific-performance","display_name":"task-specific performance","score":0.4180000126361847}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7242000102996826},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.7166000008583069},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.694100022315979},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6607000231742859},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.6209999918937683},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.61080002784729},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.5410000085830688},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.5142999887466431},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5131000280380249},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.4528000056743622},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.4020000100135803},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.3840999901294708},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38260000944137573},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3100000023841858},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3010999858379364},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.29789999127388},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.28700000047683716},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2851000130176544},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.27630001306533813},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.26570001244544983}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:arXiv.org:2610.21294","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.21294","pdf_url":"https://arxiv.org/pdf/2610.21294","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.21294","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.21294","pdf_url":"https://arxiv.org/pdf/2610.21294","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7222852263.pdf","grobid_xml":"https://content.openalex.org/works/W7222852263.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W3195140297","https://openalex.org/W3198002980","https://openalex.org/W4226146865","https://openalex.org/W4297412003","https://openalex.org/W4308023630","https://openalex.org/W4401306886","https://openalex.org/W4402955064","https://openalex.org/W4405655184","https://openalex.org/W7133225605","https://openalex.org/W7133827644"],"related_works":["https://openalex.org/W7133827644","https://openalex.org/W4386576772","https://openalex.org/W3010476030","https://openalex.org/W7222571254","https://openalex.org/W4403885496","https://openalex.org/W7133786314","https://openalex.org/W4389520094","https://openalex.org/W3156165420","https://openalex.org/W3034256339","https://openalex.org/W2946794439"],"abstract_inverted_index":{"Transformer-based":[0],"language":[1],"models":[2,99],"have":[3],"shown":[4],"strong":[5],"capabilities":[6],"across":[7,98],"many":[8,31],"tasks":[9,164],"and":[10,75,112,147,162,168],"domains.":[11],"However,":[12],"prior":[13],"work":[14],"shows":[15],"that,":[16],"for":[17,220],"a":[18,42,64,73,76,81,104,139,156,206,215],"given":[19,57],"task,":[20,77],"not":[21],"all":[22],"attention":[23,160],"heads":[24,33,85,111,191],"matter":[25],"equally.":[26],"Specifically,":[27],"they":[28],"suggest":[29],"that":[30,108,170,188],"\\textit{redundant}":[32],"can":[34,141],"be":[35],"safely":[36],"removed":[37],"with":[38,165],"little-to-no":[39],"effect,":[40],"while":[41,197],"small":[43,82,174],"set":[44,83],"of":[45,84,209],"\\textit{important}":[46],"heads,":[47],"when":[48],"ablated,":[49],"severely":[50],"damages":[51],"the":[52,56,89,179,199,210],"model's":[53],"performance":[54,126,193],"on":[55,194],"task.":[58],"In":[59],"this":[60,95,152],"work,":[61],"we":[62,117,134,154],"investigate":[63,94],"complementary":[65],"aspect,":[66],"called":[67],"\\textit{Capability":[68],"Enhancement":[69],"Localization}":[70],"(CEL):":[71],"\\textit{given":[72],"model":[74,161],"does":[78],"there":[79],"exist":[80],"whose":[86,122],"removal":[87,123],"improves":[88,124,192],"task":[90,196],"performance?}":[91],"We":[92,185],"first":[93],"question":[96],"\\textit{empirically}":[97],"from":[100,144,173],"different":[101],"families.":[102],"Using":[103],"simple":[105],"two-stage":[106],"routine":[107],"screens":[109],"candidate":[110],"evaluates":[113],"their":[114],"joint":[115,211],"ablations,":[116],"identify":[118],"sparse":[119],"head":[120],"combinations":[121],"task-specific":[125],"without":[127],"further":[128,186],"fine-tuning.":[129],"Motivated":[130],"by":[131],"these":[132],"observations,":[133],"ask":[135],"\\textit{theoretically":[136],"whether":[137],"such":[138,190],"phenomenon":[140],"naturally":[142],"arise":[143],"training":[145,203],"dynamics,":[146],"under":[148],"what":[149],"conditions}.":[150],"To":[151],"end,":[153],"construct":[155],"simplified":[157],"multi-head":[158],"softmax":[159],"two":[163],"conflicting":[166,180],"features,":[167],"show":[169,187],"gradient":[171],"descent":[172],"random":[175],"initialization":[176],"provably":[177],"concentrates":[178],"features":[181],"into":[182],"particular":[183],"heads.":[184],"ablating":[189],"one":[195],"harming":[198],"other,":[200],"even":[201],"though":[202],"converges":[204],"to":[205],"global":[207],"optimum":[208],"objective.":[212],"This":[213],"establishes":[214],"possible":[216],"training-induced":[217],"theoretical":[218],"explanation":[219],"CEL.":[221]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-18T00:00:00"}