{"id":"https://openalex.org/W4388964487","doi":"https://doi.org/10.48550/arxiv.2311.12997","title":"Compositional Capabilities of Autoregressive Transformers: A Study on Synthetic, Interpretable Tasks","display_name":"Compositional Capabilities of Autoregressive Transformers: A Study on Synthetic, Interpretable Tasks","alternate_titles":[],"publication_year":2023,"publication_date":"2023-11-21","ids":{"openalex":"https://openalex.org/W4388964487","doi":"https://doi.org/10.48550/arxiv.2311.12997"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2311.12997","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.12997","pdf_url":"https://arxiv.org/pdf/2311.12997","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4388964487"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2311.12997","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085453702","display_name":"Rahul Ramesh","orcid":"https://orcid.org/0009-0009-0134-6883","observed_orcids":["https://orcid.org/0009-0009-0134-6883"]},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Ramesh, Rahul","raw_affiliation_strings":["Computer and Information Science , University of Pennsylvania ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer and Information Science , University of Pennsylvania ,","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069090559","display_name":"Ekdeep Singh Lubana","orcid":"https://orcid.org/0000-0002-7200-9341","observed_orcids":["https://orcid.org/0000-0002-7200-9341"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]},{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lubana, Ekdeep Singh","raw_affiliation_strings":["Center for Brain Science , Harvard University ,","Electrical Engineering and Computer Science , University of Michigan ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Brain Science , Harvard University ,","institution_ids":["https://openalex.org/I136199984"]},{"raw_affiliation_string":"Electrical Engineering and Computer Science , University of Michigan ,","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025224995","display_name":"Mikail Khona","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Khona, Mikail","raw_affiliation_strings":["Physics , MIT ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Physics , MIT ,","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068224304","display_name":"Robert P. Dick","orcid":"https://orcid.org/0000-0001-5428-9530","observed_orcids":["https://orcid.org/0000-0001-5428-9530"]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Dick, Robert P.","raw_affiliation_strings":["Electrical Engineering and Computer Science , University of Michigan ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Electrical Engineering and Computer Science , University of Michigan ,","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101548632","display_name":"Hidenori Tanaka","orcid":"https://orcid.org/0000-0001-5110-3489","observed_orcids":["https://orcid.org/0000-0001-5110-3489"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]},{"id":"https://openalex.org/I2251713219","display_name":"NTT (Japan)","ror":"https://ror.org/00berct97","country_code":"JP","type":"company","lineage":["https://openalex.org/I2251713219"]}],"countries":["JP","US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Tanaka, Hidenori","raw_affiliation_strings":["Center for Brain Science , Harvard University ,","Physics & Information Labotratories , NTT Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Brain Science , Harvard University ,","institution_ids":["https://openalex.org/I136199984"]},{"raw_affiliation_string":"Physics & Information Labotratories , NTT Research","institution_ids":["https://openalex.org/I2251713219"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":25,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8583999872207642,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8583999872207642,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0746999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.01489999983459711,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/compositional-generalization","display_name":"compositional generalization","score":0.765999972820282},{"id":"https://openalex.org/keywords/autoregressive-transformer","display_name":"autoregressive transformers","score":0.9010000228881836},{"id":"https://openalex.org/keywords/synthetic-data-generation","display_name":"synthetic data generation","score":0.703000009059906},{"id":"https://openalex.org/keywords/function-composition","display_name":"function composition","score":0.7289999723434448},{"id":"https://openalex.org/keywords/intermediate-outputs","display_name":"intermediate outputs","score":0.5379999876022339},{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.4659999907016754},{"id":"https://openalex.org/keywords/training-data-bias","display_name":"training data bias","score":0.5249999761581421},{"id":"https://openalex.org/keywords/interpretability","display_name":"interpretability","score":0.5249999761581421}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6290716528892517},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4733419716358185},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.18456318974494934},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.11795184016227722},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0609891414642334}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2311.12997","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.12997","pdf_url":"https://arxiv.org/pdf/2311.12997","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2311.12997","doi":"https://doi.org/10.48550/arxiv.2311.12997","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2311.12997","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2311.12997","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.12997","pdf_url":"https://arxiv.org/pdf/2311.12997","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G5770988487","display_name":"CNS Core: Small: Importance-Aware Compressive Inference for Efficient Embedded Vision","funder_award_id":"2008151","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309370","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4388964487.pdf","grobid_xml":"https://content.openalex.org/works/W4388964487.grobid-xml"},"referenced_works_count":88,"referenced_works":["https://openalex.org/W1514897281","https://openalex.org/W2118373646","https://openalex.org/W2137083029","https://openalex.org/W2514713644","https://openalex.org/W2570615383","https://openalex.org/W2804339109","https://openalex.org/W2899663614","https://openalex.org/W2925907129","https://openalex.org/W2953307569","https://openalex.org/W2972020530","https://openalex.org/W2983486364","https://openalex.org/W3010694149","https://openalex.org/W3017374003","https://openalex.org/W3019142305","https://openalex.org/W3030163527","https://openalex.org/W3086249591","https://openalex.org/W3088599783","https://openalex.org/W3091917188","https://openalex.org/W3093233911","https://openalex.org/W3095645723","https://openalex.org/W3128590981","https://openalex.org/W3128912454","https://openalex.org/W3134678353","https://openalex.org/W3151623507","https://openalex.org/W3172314079","https://openalex.org/W3177813494","https://openalex.org/W3184144760","https://openalex.org/W3191727383","https://openalex.org/W3195140297","https://openalex.org/W3195577433","https://openalex.org/W3197009789","https://openalex.org/W3198002980","https://openalex.org/W3201174429","https://openalex.org/W3205068155","https://openalex.org/W3205643961","https://openalex.org/W3207316473","https://openalex.org/W3215529831","https://openalex.org/W4221151371","https://openalex.org/W4224308101","https://openalex.org/W4226112058","https://openalex.org/W4226146865","https://openalex.org/W4226271906","https://openalex.org/W4226399820","https://openalex.org/W4281690148","https://openalex.org/W4283026156","https://openalex.org/W4283157303","https://openalex.org/W4285484119","https://openalex.org/W4297412003","https://openalex.org/W4302305823","https://openalex.org/W4302307977","https://openalex.org/W4306808908","https://openalex.org/W4308023630","https://openalex.org/W4316135772","https://openalex.org/W4317466024","https://openalex.org/W4318348216","https://openalex.org/W4319453780","https://openalex.org/W4319991279","https://openalex.org/W4320003957","https://openalex.org/W4322718191","https://openalex.org/W4360836968","https://openalex.org/W4366330503","https://openalex.org/W4376632242","https://openalex.org/W4378474292","https://openalex.org/W4385473649","https://openalex.org/W4385474529","https://openalex.org/W4385571789","https://openalex.org/W4385573190","https://openalex.org/W4385573565","https://openalex.org/W4385573838","https://openalex.org/W4387634967","https://openalex.org/W4387947637","https://openalex.org/W4387994687","https://openalex.org/W4389518382","https://openalex.org/W4389524426","https://openalex.org/W4410356661","https://openalex.org/W4410357125","https://openalex.org/W4410357454","https://openalex.org/W7058409103","https://openalex.org/W7126382208","https://openalex.org/W7133195599","https://openalex.org/W7133217671","https://openalex.org/W7133219841","https://openalex.org/W7133224126","https://openalex.org/W7133225605","https://openalex.org/W7133226766","https://openalex.org/W7133227602","https://openalex.org/W7133231369","https://openalex.org/W7133240241"],"related_works":["https://openalex.org/W4403749603","https://openalex.org/W4407571503","https://openalex.org/W7163073884","https://openalex.org/W3213214466","https://openalex.org/W3191727383","https://openalex.org/W7162646996","https://openalex.org/W6989908802","https://openalex.org/W7127565955","https://openalex.org/W6929485549","https://openalex.org/W4415799289"],"abstract_inverted_index":{"Transformers":[0,89,145],"trained":[1],"on":[2,45,57,80],"huge":[3],"text":[4],"corpora":[5],"exhibit":[6],"a":[7,36,58,66,73],"remarkable":[8],"set":[9,67],"of":[10,21,39,65,68,75,97,136,152],"capabilities,":[11,33],"e.g.,":[12],"performing":[13],"basic":[14],"arithmetic.":[15],"Given":[16],"the":[17,26,50,134,137,140,156,165,169],"inherent":[18],"compositional":[19,92],"nature":[20],"language,":[22],"one":[23],"can":[24,43,90],"expect":[25],"model":[27],"to":[28,30,102,121,148,162],"learn":[29,91],"compose":[31,149],"these":[32],"potentially":[34],"yielding":[35],"combinatorial":[37],"explosion":[38],"what":[40],"operations":[41],"it":[42],"perform":[44],"an":[46],"input.":[47],"Motivated":[48],"by":[49],"above,":[51],"we":[52,84],"train":[53],"autoregressive":[54,88],"Transformer":[55],"models":[56],"synthetic":[59],"data-generating":[60,82],"process":[61],"that":[62,146],"involves":[63],"compositions":[64,124,138],"well-defined":[69],"monolithic":[70],"capabilities.":[71],"Through":[72],"series":[74],"extensive":[76],"and":[77,100,154],"systematic":[78],"experiments":[79],"this":[81],"process,":[83],"show":[85],"that:":[86],"(1)":[87],"structures":[93],"from":[94],"small":[95],"amounts":[96],"training":[98,141],"data":[99,142],"generalize":[101],"exponentially":[103],"or":[104],"even":[105],"combinatorially":[106],"many":[107],"functions;":[108,153],"(2)":[109],"generating":[110,127],"intermediate":[111,129],"outputs":[112,130],"when":[113],"composing":[114],"functions":[115],"is":[116],"more":[117],"effective":[118],"for":[119],"generalizing":[120],"new,":[122],"unseen":[123],"than":[125],"not":[126],"any":[128],"(3)":[131],"biases":[132],"in":[133,139,144],"order":[135],"result":[143],"fail":[147],"some":[150],"combinations":[151],"(4)":[155],"attention":[157],"layers":[158,167],"select":[159],"which":[160],"capability":[161],"apply":[163],"while":[164],"feed-forward":[166],"execute":[168],"selected":[170],"capability.":[171]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4388964487","counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2023-11-24T00:00:00"}
