{"id":"https://openalex.org/W4389599535","doi":"https://doi.org/10.48550/arxiv.2312.04927","title":"Zoology: Measuring and Improving Recall in Efficient Language Models","display_name":"Zoology: Measuring and Improving Recall in Efficient Language Models","publication_year":2023,"publication_date":"2023-12-08","ids":{"openalex":"https://openalex.org/W4389599535","doi":"https://doi.org/10.48550/arxiv.2312.04927"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2312.04927","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.04927","pdf_url":"https://arxiv.org/pdf/2312.04927","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2312.04927","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Arora, Simran","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arora, Simran","raw_affiliation_strings":["University at Buffalo \u25b3 Purdue University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University at Buffalo \u25b3 Purdue University","institution_ids":["https://openalex.org/I219193219","https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011478764","display_name":"Sabri Eyuboglu","orcid":"https://orcid.org/0000-0002-8412-0266","observed_orcids":["https://orcid.org/0000-0002-8412-0266"]},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eyuboglu, Sabri","raw_affiliation_strings":["University at Buffalo \u25b3 Purdue University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University at Buffalo \u25b3 Purdue University","institution_ids":["https://openalex.org/I219193219","https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036729848","display_name":"Aman Timalsina","orcid":"https://orcid.org/0000-0002-2854-4208","observed_orcids":["https://orcid.org/0000-0002-2854-4208"]},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Timalsina, Aman","raw_affiliation_strings":["University at Buffalo \u25b3 Purdue University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University at Buffalo \u25b3 Purdue University","institution_ids":["https://openalex.org/I219193219","https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024441871","display_name":"Isys Johnson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Johnson, Isys","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078213488","display_name":"Michael Poli","orcid":"https://orcid.org/0000-0001-5384-9372","observed_orcids":["https://orcid.org/0000-0001-5384-9372"]},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Poli, Michael","raw_affiliation_strings":["University at Buffalo \u25b3 Purdue University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University at Buffalo \u25b3 Purdue University","institution_ids":["https://openalex.org/I219193219","https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005779176","display_name":"James Zou","orcid":"https://orcid.org/0000-0001-8880-4764","observed_orcids":["https://orcid.org/0000-0001-8880-4764"]},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zou, James","raw_affiliation_strings":["University at Buffalo \u25b3 Purdue University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University at Buffalo \u25b3 Purdue University","institution_ids":["https://openalex.org/I219193219","https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001041485","display_name":"Atri Rudra B","orcid":"https://orcid.org/0000-0003-4136-4719","observed_orcids":["https://orcid.org/0000-0003-4136-4719"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rudra, Atri","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103852640","display_name":"Christopher R\u00e9","orcid":"https://orcid.org/0000-0002-4109-0108","observed_orcids":["https://orcid.org/0000-0002-4109-0108"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"R\u00e9, Christopher","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8598999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8598999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.10909999907016754,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/gated-convolution","display_name":"gated convolution","score":0.5960000157356262},{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.7210000157356262},{"id":"https://openalex.org/keywords/associative-recall","display_name":"associative recall","score":0.9490000009536743},{"id":"https://openalex.org/keywords/parameter-efficiency","display_name":"parameter efficiency","score":0.6380000114440918},{"id":"https://openalex.org/keywords/sparse-attention","display_name":"sparse attention","score":0.8309999704360962}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7638975381851196},{"id":"https://openalex.org/C45347329","wikidata":"https://www.wikidata.org/wiki/Q5166604","display_name":"Convolution (computer science)","level":3,"score":0.6922639608383179},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5787901878356934},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5606090426445007},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.5456032752990723},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5031124949455261},{"id":"https://openalex.org/C159423971","wikidata":"https://www.wikidata.org/wiki/Q177251","display_name":"Associative property","level":2,"score":0.4927857220172882},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.48796504735946655},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4315919280052185},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.415225088596344},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3676643967628479},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.35793018341064453},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.18146738409996033},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1113385260105133},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.07445979118347168},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2312.04927","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.04927","pdf_url":"https://arxiv.org/pdf/2312.04927","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2312.04927","doi":"https://doi.org/10.48550/arxiv.2312.04927","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2312.04927","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2312.04927","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.04927","pdf_url":"https://arxiv.org/pdf/2312.04927","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"score":0.5799999833106995,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G1447151805","display_name":"AF: Medium: Collaborative Research: Beyond Sparsity: Refined Measures of Complexity for Linear Algebra","funder_award_id":"1763315","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G1712871328","display_name":null,"funder_award_id":"FA86501827882","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G2040692823","display_name":"SHF: Medium: Collaborative Research: From Volume to Velocity: Big Data Analytics in Near-Realtime","funder_award_id":"1563078","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2589055757","display_name":null,"funder_award_id":"N000141712266","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"},{"id":"https://openalex.org/G2935333310","display_name":null,"funder_award_id":"N000141712266","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G538509161","display_name":null,"funder_award_id":"FA86501827865","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G7568964499","display_name":"RTML: Large: Continuous Adaptation for Decision Streams","funder_award_id":"1937301","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G789143534","display_name":"Mobility Data Integration to Insight","funder_award_id":"U54EB020405","funder_id":"https://openalex.org/F4320337363","funder_display_name":"National Institute of Biomedical Imaging and Bioengineering"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306202","display_name":"Gordon and Betty Moore Foundation","ror":"https://ror.org/006wxqw41"},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320337363","display_name":"National Institute of Biomedical Imaging and Bioengineering","ror":"https://ror.org/00372qc85"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389599535.pdf","grobid_xml":"https://content.openalex.org/works/W4389599535.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2963970792","https://openalex.org/W4401157551","https://openalex.org/W7166867524","https://openalex.org/W7221763987","https://openalex.org/W7169835299","https://openalex.org/W7220745173","https://openalex.org/W7213409806","https://openalex.org/W7161697858","https://openalex.org/W4409364153","https://openalex.org/W4411119666"],"abstract_inverted_index":{"Attention-free":[0],"language":[1,34],"models":[2],"that":[3,37,71,97,159,173,195],"combine":[4],"gating":[5],"and":[6,16,32,143,168,181,193],"convolutions":[7,128],"are":[8],"growing":[9],"in":[10,176],"popularity":[11],"due":[12],"to":[13,46,68,208],"their":[14],"efficiency":[15],"increasingly":[17],"competitive":[18],"performance.":[19],"To":[20,137],"better":[21,160],"understand":[22],"these":[23],"architectures,":[24],"we":[25,56,95,146,188],"pretrain":[26],"a":[27,102,105,111,148],"suite":[28],"of":[29,59,151,171,179,205],"17":[30],"attention":[31,43,98,108,180,200],"\"gated-convolution\"":[33],"models,":[35],"finding":[36],"SoTA":[38],"gated-convolution":[39,115,182],"architectures":[40],"still":[41],"underperform":[42],"by":[44,64,101,185],"up":[45],"2.1":[47],"perplexity":[48],"points":[49],"on":[50,117],"the":[51,60,139,152,177,206],"Pile.":[52],"In":[53],"fine-grained":[54],"analysis,":[55,187],"find":[57,96],"82%":[58],"gap":[61,140,207],"is":[62,72,121,216],"explained":[63],"each":[65],"model's":[66],"ability":[67],"recall":[69,157],"information":[70],"previously":[73],"mentioned":[74],"in-context,":[75],"e.g.":[76],"\"Hakuna":[77],"Matata":[78,83],"means":[79,85],"no":[80],"worries":[81],"Hakuna":[82],"it":[84],"no\"":[86],"$\\rightarrow$":[87],"\"??\".":[88],"On":[89],"this":[90],"task,":[91],"termed":[92],"\"associative":[93],"recall\",":[94],"outperforms":[99,110],"gated-convolutions":[100],"large":[103],"margin:":[104],"70M":[106],"parameter":[107,114],"model":[109,116],"1.4":[112],"billion":[113],"associative":[118,156],"recall.":[119,183],"This":[120],"surprising":[122],"because":[123],"prior":[124],"work":[125],"shows":[126],"gated":[127],"can":[129,202],"perfectly":[130],"solve":[131],"synthetic":[132],"tests":[133],"for":[134],"AR":[135],"capability.":[136],"close":[138,203],"between":[141],"synthetics":[142],"real":[144],"language,":[145],"develop":[147],"new":[149],"formalization":[150],"task":[153],"called":[154],"multi-query":[155],"(MQAR)":[158],"reflects":[161],"actual":[162],"language.":[163],"We":[164],"perform":[165],"an":[166],"empirical":[167],"theoretical":[169],"study":[170],"MQAR":[172],"elucidates":[174],"differences":[175],"parameter-efficiency":[178],"Informed":[184],"our":[186],"evaluate":[189],"simple":[190],"convolution-attention":[191],"hybrids":[192,196],"show":[194],"with":[197],"input-dependent":[198],"sparse":[199],"patterns":[201],"97.4%":[204],"attention,":[209],"while":[210],"maintaining":[211],"sub-quadratic":[212],"scaling.":[213],"Our":[214],"code":[215],"accessible":[217],"at:":[218],"https://github.com/HazyResearch/zoology.":[219]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2023-12-12T00:00:00"}