{"id":"https://openalex.org/W4400518737","doi":"https://doi.org/10.48550/arxiv.2407.06488","title":"Towards Understanding Multi-Task Learning (Generalization) of LLMs via Detecting and Exploring Task-Specific Neurons","display_name":"Towards Understanding Multi-Task Learning (Generalization) of LLMs via Detecting and Exploring Task-Specific Neurons","publication_year":2024,"publication_date":"2024-07-09","ids":{"openalex":"https://openalex.org/W4400518737","doi":"https://doi.org/10.48550/arxiv.2407.06488"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.06488","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.06488","pdf_url":"https://arxiv.org/pdf/2407.06488","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.06488","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101253394","display_name":"Yongqi Leng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Leng, Yongqi","raw_affiliation_strings":["College of Intelligence and Computing , Tianjin University , Tianjin , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing , Tianjin University , Tianjin , China","institution_ids":["https://openalex.org/I162868743"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5055232825","display_name":"Deyi Xiong","orcid":"https://orcid.org/0000-0002-2353-5038","observed_orcids":["https://orcid.org/0000-0002-2353-5038","https://orcid.org/0009-0007-6501-4496"]},"institutions":[{"id":"https://openalex.org/I162868743","display_name":"Tianjin University","ror":"https://ror.org/012tb2g32","country_code":"CN","type":"education","lineage":["https://openalex.org/I162868743"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiong, Deyi","raw_affiliation_strings":["College of Intelligence and Computing , Tianjin University , Tianjin , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Intelligence and Computing , Tianjin University , Tianjin , China","institution_ids":["https://openalex.org/I162868743"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I162868743"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.8628000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.8628000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.07180000096559525,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.04190000146627426,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"multi-task learning","score":0.925000011920929},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8989999890327454},{"id":"https://openalex.org/keywords/catastrophic-forgetting","display_name":"catastrophic forgetting","score":0.9679999947547913},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.7850000262260437},{"id":"https://openalex.org/keywords/continuous-learning","display_name":"continuous learning","score":0.8629999756813049},{"id":"https://openalex.org/keywords/interpretability","display_name":"interpretability","score":0.7289999723434448}],"concepts":[{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.801831841468811},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.7887319922447205},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4321031868457794},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4104631841182709},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3755950927734375},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3475855588912964},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.09290242195129395},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.07850933074951172},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.05827650427818298},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.06488","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.06488","pdf_url":"https://arxiv.org/pdf/2407.06488","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.06488","doi":"https://doi.org/10.48550/arxiv.2407.06488","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.06488","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.06488","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.06488","pdf_url":"https://arxiv.org/pdf/2407.06488","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400518737.pdf","grobid_xml":"https://content.openalex.org/works/W4400518737.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4404782916","https://openalex.org/W7120071406","https://openalex.org/W7140373145","https://openalex.org/W7162435025","https://openalex.org/W4392974112","https://openalex.org/W4405792812","https://openalex.org/W4414691182","https://openalex.org/W4399695807","https://openalex.org/W4404782464","https://openalex.org/W4416034935"],"abstract_inverted_index":{"While":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"have":[5],"demonstrated":[6],"superior":[7],"multi-task":[8,85,180],"capabilities,":[9],"understanding":[10],"the":[11,31,57,64,97,123,136,153,164,167,175],"learning":[12,86],"mechanisms":[13,29],"behind":[14],"this":[15,22],"is":[16,102,118,132],"still":[17],"a":[18,119,145],"challenging":[19],"problem.":[20],"In":[21],"paper,":[23],"we":[24,36,54,68,78,143],"attempt":[25],"to":[26],"understand":[27],"such":[28,130],"from":[30],"perspective":[32],"of":[33,99,115,125,166,177],"neurons.":[34,72],"Specifically,":[35],"detect":[37],"task-sensitive":[38],"neurons":[39,59,101,156],"in":[40,84,122,179],"LLMs":[41,178],"via":[42],"gradient":[43],"attribution":[44],"on":[45],"task-specific":[46,71,76,100,127,155],"data.":[47],"Through":[48],"extensive":[49,161],"deactivation":[50],"and":[51,87,91,107,129,160],"fine-tuning":[52,148],"experiments,":[53],"demonstrate":[55,163],"that":[56,96,150],"detected":[58],"are":[60],"highly":[61,133],"correlated":[62,134],"with":[63,105,135],"given":[65],"task,":[66],"which":[67],"term":[69],"as":[70],"With":[73],"these":[74,141],"identified":[75],"neurons,":[77,128],"delve":[79],"into":[80,174],"two":[81],"common":[82],"problems":[83],"continuous":[88,147,158],"learning:":[89],"Generalization":[90],"Catastrophic":[92],"Forgetting.":[93],"We":[94],"find":[95],"overlap":[98],"strongly":[103],"associated":[104],"generalization":[106,137],"specialization":[108],"across":[109],"tasks.":[110],"Interestingly,":[111],"at":[112],"certain":[113],"layers":[114],"LLMs,":[116],"there":[117],"high":[120],"similarity":[121,131],"parameters":[124],"different":[126],"performance.":[138],"Inspired":[139],"by":[140],"findings,":[142],"propose":[144],"neuron-level":[146],"method":[149],"only":[151],"fine-tunes":[152],"current":[154],"during":[157],"learning,":[159],"experiments":[162],"effectiveness":[165],"proposed":[168],"method.":[169],"Our":[170],"study":[171],"provides":[172],"insights":[173],"interpretability":[176],"learning.":[181]},"counts_by_year":[],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2024-07-11T00:00:00"}