{"id":"https://openalex.org/W4221151371","doi":"https://doi.org/10.48550/arxiv.2202.07206","title":"Impact of Pretraining Term Frequencies on Few-Shot Reasoning","display_name":"Impact of Pretraining Term Frequencies on Few-Shot Reasoning","publication_year":2022,"publication_date":"2022-02-15","ids":{"openalex":"https://openalex.org/W4221151371","doi":"https://doi.org/10.48550/arxiv.2202.07206"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2202.07206","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2202.07206","pdf_url":"https://arxiv.org/pdf/2202.07206","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2202.07206","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003112970","display_name":"Yasaman Razeghi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Razeghi, Yasaman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103983469","display_name":"Robert L. Logan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Logan, Robert L.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035088083","display_name":"Matt Gardner","orcid":"https://orcid.org/0000-0001-8458-1727","observed_orcids":["https://orcid.org/0000-0001-8458-1727"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gardner, Matt","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5005779128","display_name":"Sameer Singh","orcid":"https://orcid.org/0000-0003-0621-6323","observed_orcids":["https://orcid.org/0000-0003-0621-6323"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Singh, Sameer","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":139,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9204000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9204000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.035100001841783524,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.02459999918937683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/numerical-reasoning","display_name":"numerical reasoning","score":0.8690000176429749},{"id":"https://openalex.org/keywords/term-frequency","display_name":"term frequency","score":0.8809999823570251},{"id":"https://openalex.org/keywords/pre-training-data","display_name":"pre-training data","score":0.8849999904632568},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.5709999799728394},{"id":"https://openalex.org/keywords/arithmetic","display_name":"arithmetic","score":0.7319999933242798},{"id":"https://openalex.org/keywords/unit-conversion","display_name":"unit conversion","score":0.9950000047683716},{"id":"https://openalex.org/keywords/extrapolation","display_name":"extrapolation","score":0.4050000011920929}],"concepts":[{"id":"https://openalex.org/C132459708","wikidata":"https://www.wikidata.org/wiki/Q744069","display_name":"Extrapolation","level":2,"score":0.8548057079315186},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.6668163537979126},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.6589543223381042},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6217997074127197},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.5552107691764832},{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.4897732734680176},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4466722011566162},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34453001618385315},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3322080075740814},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3320436477661133},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2818434238433838},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.27900493144989014},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.18818610906600952},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2202.07206","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2202.07206","pdf_url":"https://arxiv.org/pdf/2202.07206","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2202.07206","doi":"https://doi.org/10.48550/arxiv.2202.07206","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2202.07206","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2202.07206","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2202.07206","pdf_url":"https://arxiv.org/pdf/2202.07206","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G5865625667","display_name":null,"funder_award_id":"N660011924033","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G6416336507","display_name":"CAREER: Detecting, Understanding, and Fixing Vulnerabilities in Natural Language Processing Models","funder_award_id":"2046873","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4221151371.pdf","grobid_xml":"https://content.openalex.org/works/W4221151371.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W2790415926","https://openalex.org/W2798280648","https://openalex.org/W2798358706","https://openalex.org/W2804713279","https://openalex.org/W2889453388","https://openalex.org/W2951243568","https://openalex.org/W2962809918","https://openalex.org/W2963969878","https://openalex.org/W2964183365","https://openalex.org/W2970161131","https://openalex.org/W2970476646","https://openalex.org/W2974596953","https://openalex.org/W3015339775","https://openalex.org/W3020979945","https://openalex.org/W3030163527","https://openalex.org/W3034602344","https://openalex.org/W3035139434","https://openalex.org/W3047026265","https://openalex.org/W3088599783","https://openalex.org/W3091922147","https://openalex.org/W3098267758","https://openalex.org/W3098757341","https://openalex.org/W3118781290","https://openalex.org/W3128551234","https://openalex.org/W3152698349","https://openalex.org/W3155431862","https://openalex.org/W3158635868","https://openalex.org/W3158822421","https://openalex.org/W3176693244","https://openalex.org/W3195577433","https://openalex.org/W3200809495","https://openalex.org/W3212496002","https://openalex.org/W4385567149","https://openalex.org/W7133260402"],"related_works":["https://openalex.org/W4385573636","https://openalex.org/W2998696444","https://openalex.org/W4385573037","https://openalex.org/W3015339775","https://openalex.org/W7071565425","https://openalex.org/W4385571976","https://openalex.org/W4305009356","https://openalex.org/W4414968597","https://openalex.org/W7163385082","https://openalex.org/W4404574036"],"abstract_inverted_index":{"Pretrained":[0],"Language":[1],"Models":[2],"(LMs)":[3],"have":[4],"demonstrated":[5],"ability":[6],"to":[7,23,139,172],"perform":[8],"numerical":[9,99,151],"reasoning":[10,30,152],"by":[11],"extrapolating":[12],"from":[13,71],"a":[14,86],"few":[15],"examples":[16],"in":[17,49,74,123,137],"few-shot":[18,150],"settings.":[19],"However,":[20],"the":[21,50,57,60,67,75,80,94,132,140,157,174],"extent":[22],"which":[24],"this":[25,34,83],"extrapolation":[26],"relies":[27],"on":[28,63,93,97,116,131],"robust":[29],"is":[31],"unclear.":[32],"In":[33,53],"paper,":[35],"we":[36,55,169],"investigate":[37],"how":[38,160],"well":[39],"these":[40],"models":[41,91,112,162],"reason":[42],"with":[43],"terms":[44,70,119,136],"that":[45,111],"are":[46,113,120],"less":[47],"frequent":[48,135],"pretraining":[51,76,166,175],"data.":[52,77],"particular,":[54],"examine":[56],"correlations":[58],"between":[59],"model":[61],"performance":[62,148],"test":[64],"instances":[65,73,117],"and":[66,104,168],"frequency":[68],"of":[69,82,88,159],"those":[72],"We":[78],"measure":[79],"strength":[81],"correlation":[84],"for":[85],"number":[87],"GPT-based":[89],"language":[90],"(pretrained":[92],"Pile":[95],"dataset)":[96],"various":[98],"deduction":[100],"tasks":[101],"(e.g.,":[102],"arithmetic":[103],"unit":[105],"conversion).":[106],"Our":[107],"results":[108,155],"consistently":[109],"demonstrate":[110],"more":[114,121,129],"accurate":[115,130],"whose":[118],"prevalent,":[122],"some":[124],"cases":[125],"above":[126],"$70\\%$":[127],"(absolute)":[128],"top":[133],"10\\%":[134],"comparison":[138],"bottom":[141],"10\\%.":[142],"Overall,":[143],"although":[144],"LMs":[145],"exhibit":[146],"strong":[147],"at":[149],"tasks,":[153],"our":[154],"raise":[156],"question":[158],"much":[161],"actually":[163],"generalize":[164],"beyond":[165],"data,":[167],"encourage":[170],"researchers":[171],"take":[173],"data":[176],"into":[177],"account":[178],"when":[179],"interpreting":[180],"evaluation":[181],"results.":[182]},"counts_by_year":[{"year":2026,"cited_by_count":11},{"year":2025,"cited_by_count":26},{"year":2024,"cited_by_count":26},{"year":2023,"cited_by_count":51},{"year":2022,"cited_by_count":25}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2022-04-03T00:00:00"}