{"id":"https://openalex.org/W4389518992","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.503","title":"How Predictable Are Large Language Model Capabilities? A Case Study on BIG-bench","display_name":"How Predictable Are Large Language Model Capabilities? A Case Study on BIG-bench","alternate_titles":[],"publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389518992","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.503"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.503","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.503","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.503","pdf_url":"https://aclanthology.org/2023.findings-emnlp.503.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4389518992"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.findings-emnlp.503.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022956157","display_name":"Qinyuan Ye","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Qinyuan Ye","raw_affiliation_strings":["University of Southern California, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100580799","display_name":"Harvey Yiyun Fu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Harvey Fu","raw_affiliation_strings":["University of Southern California, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009408707","display_name":"Xiang Ren","orcid":"https://orcid.org/0000-0001-8655-663X","observed_orcids":["https://orcid.org/0000-0001-8655-663X"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Xiang Ren","raw_affiliation_strings":["University of Southern California, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041906762","display_name":"Robin Jia","orcid":"https://orcid.org/0009-0002-8123-7132","observed_orcids":["https://orcid.org/0009-0002-8123-7132"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Robin Jia","raw_affiliation_strings":["University of Southern California, Los Angeles, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1174212"],"apc_list":null,"apc_paid":null,"fwci":2.9147,"has_fulltext":true,"cited_by_count":50,"citation_normalized_percentile":{"value":0.94342345,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":92,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"7493","last_page":"7517"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9223999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9223999977111816,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.02590000070631504,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.023900000378489494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8880000114440918},{"id":"https://openalex.org/keywords/performance-prediction","display_name":"performance prediction","score":0.8889999985694885},{"id":"https://openalex.org/keywords/big-bench","display_name":"BIG-Bench","score":0.902999997138977},{"id":"https://openalex.org/keywords/task-variety","display_name":"task variety","score":0.6169999837875366},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.6850000023841858},{"id":"https://openalex.org/keywords/model-families","display_name":"model families","score":0.4020000100135803},{"id":"https://openalex.org/keywords/task-clustering","display_name":"task clustering","score":0.6549999713897705}],"concepts":[{"id":"https://openalex.org/C197640229","wikidata":"https://www.wikidata.org/wiki/Q2534066","display_name":"Predictability","level":2,"score":0.7367293834686279},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.735149085521698},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.6107559204101562},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6095484495162964},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5880361795425415},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5784726142883301},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.5768611431121826},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5240141153335571},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5038041472434998},{"id":"https://openalex.org/C2776266606","wikidata":"https://www.wikidata.org/wiki/Q476482","display_name":"Test bench","level":2,"score":0.48207026720046997},{"id":"https://openalex.org/C146599234","wikidata":"https://www.wikidata.org/wiki/Q511093","display_name":"Centroid","level":2,"score":0.4689442813396454},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.42087921500205994},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3779641389846802},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33643263578414917},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.11983513832092285},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1055971086025238},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08243873715400696},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.findings-emnlp.503","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.503","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.503","pdf_url":"https://aclanthology.org/2023.findings-emnlp.503.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2305.14947","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.14947","pdf_url":"https://arxiv.org/pdf/2305.14947","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2305.14947","doi":"https://doi.org/10.48550/arxiv.2305.14947","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2305.14947","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.503","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.503","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.503","pdf_url":"https://aclanthology.org/2023.findings-emnlp.503.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1002423621","display_name":"CAREER: Teaching Machines through Human Explanation for Information Extraction","funder_award_id":"2048211","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G390747679","display_name":null,"funder_award_id":"W911NF\u201019\u20102\u20100271","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G5865625667","display_name":null,"funder_award_id":"N660011924033","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G608057831","display_name":null,"funder_award_id":"W911NF-19","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307791","display_name":"Cisco Systems","ror":"https://ror.org/03yt1ez60"},{"id":"https://openalex.org/F4320312530","display_name":"Office of the Director of National Intelligence","ror":"https://ror.org/01v3fsc55"},{"id":"https://openalex.org/F4320315389","display_name":"Open Philanthropy Project","ror":"https://ror.org/004d1k391"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320333051","display_name":"Intelligence Advanced Research Projects Activity","ror":"https://ror.org/01v3fsc55"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389518992.pdf","grobid_xml":"https://content.openalex.org/works/W4389518992.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W1840435438","https://openalex.org/W2154061444","https://openalex.org/W2295598076","https://openalex.org/W2911964244","https://openalex.org/W2923014074","https://openalex.org/W2990704537","https://openalex.org/W3030163527","https://openalex.org/W3034776473","https://openalex.org/W3154806625","https://openalex.org/W3199241049","https://openalex.org/W3212496002","https://openalex.org/W4229005866","https://openalex.org/W4281690148","https://openalex.org/W4283026156","https://openalex.org/W4308049129","https://openalex.org/W4322718191","https://openalex.org/W4385571157","https://openalex.org/W4385573444","https://openalex.org/W4411630215","https://openalex.org/W7113318674","https://openalex.org/W7133224126"],"related_works":["https://openalex.org/W4404817104","https://openalex.org/W4417134733","https://openalex.org/W4390810155","https://openalex.org/W4405255152","https://openalex.org/W7148198523","https://openalex.org/W4403556531","https://openalex.org/W4389520466","https://openalex.org/W7222949636","https://openalex.org/W6922800472","https://openalex.org/W4405296133"],"abstract_inverted_index":{"We":[0],"investigate":[1],"the":[2,60,72,97,103,108,122,125,170],"predictability":[3],"of":[4,12,20,25,99,110,117,172],"large":[5],"language":[6],"model":[7,17,143],"(LLM)":[8],"capabilities:":[9],"given":[10],"records":[11,78],"past":[13],"experiments":[14],"using":[15],"different":[16],"families,":[18,144],"numbers":[19,24],"parameters,":[21],"tasks,":[22],"and":[23,59,162],"in-context":[26],"examples,":[27],"can":[28,128],"we":[29,149],"accurately":[30],"predict":[31],"LLM":[32,44],"performance":[33,73,123],"on":[34,56,76,124],"new":[35,142],"experiment":[36,77,104],"configurations?Answering":[37],"this":[38],"question":[39],"has":[40],"practical":[41],"implications":[42],"for":[43,112,140],"users":[45],"(e.g.,":[46,53,63],"deciding":[47],"which":[48,121],"models":[49],"to":[50,166],"try),":[51],"developers":[52],"prioritizing":[54],"evaluation":[55],"representative":[57],"tasks),":[58],"research":[61],"community":[62],"identifying":[64],"hard-to-predict":[65],"capabilities":[66],"that":[67],"warrant":[68],"further":[69],"investigation).We":[70],"study":[71],"prediction":[74],"problem":[75,109],"from":[79,120],"BIG-bench.On":[80],"a":[81,133],"random":[82],"train-test":[83],"split,":[84],"an":[85,89,114],"MLP-based":[86],"predictor":[87,161],"achieves":[88],"R":[90],"2":[91],"score":[92],"greater":[93],"than":[94],"95%,":[95],"indicating":[96],"presence":[98],"learnable":[100],"patterns":[101],"within":[102],"records.We":[105],"then":[106],"formulate":[107],"searching":[111],"\"small-bench,\"":[113],"informative":[115,136],"subset":[116,134],"BIG-bench":[118,138],"tasks":[119,164],"full":[126],"set":[127],"be":[129],"maximally":[130],"recovered.We":[131],"find":[132,150],"as":[135,137],"Hard":[139],"evaluating":[141],"while":[145],"being":[146],"3\u00d7":[147],"smaller.Additionally,":[148],"competitive":[151],"subsets":[152],"by":[153,158],"clustering":[154],"task":[155,173],"representations":[156],"learned":[157],"our":[159],"MLPbased":[160],"selecting":[163],"close":[165],"cluster":[167],"centroids,":[168],"highlighting":[169],"importance":[171],"diversity":[174],"in":[175],"constructing":[176],"\"small-bench.\"":[177],"1":[178]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4389518992","counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":2}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2023-12-10T00:00:00"}
