{"id":"https://openalex.org/W6893809404","doi":"https://doi.org/10.5281/zenodo.5371628","title":"EleutherAI/lm-evaluation-harness: v0.4.13","display_name":"EleutherAI/lm-evaluation-harness: v0.4.13","alternate_titles":[],"publication_year":2021,"publication_date":"2021-09-02","ids":{"openalex":"https://openalex.org/W6893809404","doi":"https://doi.org/10.5281/zenodo.5371628"},"language":"en","primary_location":{"id":"pmh:oai:zenodo.org:5371629","doi":null,"is_oa":true,"landing_page_url":"https://zenodo.org/record/5371629","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"software"},"parent_works":[],"lineage":["https://openalex.org/W6893809404"],"child_works_count":0,"type":"software","is_traditional":false,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://zenodo.org/record/5371629","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Lintang Sutawika","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lintang Sutawika","raw_affiliation_strings":["@EleutherAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"@EleutherAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057062066","display_name":"Leo Gao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Leo Gao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080237064","display_name":"Hailey Schoelkopf","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Hailey Schoelkopf","raw_affiliation_strings":["EleutherAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"EleutherAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Stella Biderman","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1322124587","display_name":"Booz Allen Hamilton (United States)","ror":"https://ror.org/051rcp357","country_code":"US","type":"company","lineage":["https://openalex.org/I1322124587"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Stella Biderman","raw_affiliation_strings":["Booz Allen Hamilton, EleutherAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Booz Allen Hamilton, EleutherAI","institution_ids":["https://openalex.org/I1322124587"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002088849","display_name":"Jonathan Tow","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Jonathan Tow","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098824372","display_name":"Baber Abbasi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Baber Abbasi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"ben fattori","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"ben fattori","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Charles Lovering","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Charles Lovering","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"farzanehnakhaee70","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"farzanehnakhaee70","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Jason Phang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Jason Phang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026929519","display_name":"Anish Thite","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Anish Thite","raw_affiliation_strings":["@ClarosAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"@ClarosAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Fazz","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Fazz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Aflah","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I119939252","display_name":"Indraprastha Institute of Information Technology Delhi","ror":"https://ror.org/03vfp4g33","country_code":"IN","type":"education","lineage":["https://openalex.org/I119939252"]}],"countries":["IN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Aflah","raw_affiliation_strings":["Indraprastha Institute of Information Technology Delhi"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Indraprastha Institute of Information Technology Delhi","institution_ids":["https://openalex.org/I119939252"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000043237","display_name":"Niklas Muennighoff","orcid":"https://orcid.org/0009-0001-7157-770X","observed_orcids":["https://orcid.org/0009-0001-7157-770X"]},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Niklas Muennighoff","raw_affiliation_strings":["Peking University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Thomas Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4387154989","display_name":"Hugging Face","ror":"https://ror.org/02grspc61","country_code":"US","type":"company","lineage":["https://openalex.org/I4387154989"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Thomas Wang","raw_affiliation_strings":["Hugging Face"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hugging Face","institution_ids":["https://openalex.org/I4387154989"]}]},{"author_position":"middle","author":{"id":null,"display_name":"sdtblck","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"sdtblck","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"nopperl","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"nopperl","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"gakada","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"gakada","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"tttyuntian","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"tttyuntian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"researcher2","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"researcher2","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chris","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Chris","raw_affiliation_strings":["@azurro"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"@azurro","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016315389","display_name":"Julen Etxaniz","orcid":"https://orcid.org/0009-0000-2099-7766","observed_orcids":["https://orcid.org/0009-0000-2099-7766"]},"institutions":[{"id":"https://openalex.org/I169108374","display_name":"University of the Basque Country","ror":"https://ror.org/000xsnr85","country_code":"ES","type":"education","lineage":["https://openalex.org/I169108374"]}],"countries":["ES"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Julen Etxaniz","raw_affiliation_strings":["Hitz Zentroa UPV/EHU"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hitz Zentroa UPV/EHU","institution_ids":["https://openalex.org/I169108374"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032644567","display_name":"Zden\u011bk Kasner","orcid":"https://orcid.org/0000-0002-5753-5538","observed_orcids":["https://orcid.org/0000-0002-5753-5538"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Zden\u011bk Kasner","raw_affiliation_strings":["@ufal"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"@ufal","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Khalid","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Khalid","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Jeffrey Hsu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Jeffrey Hsu","raw_affiliation_strings":["Ivy Natal"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ivy Natal","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"AndyZwei","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"AndyZwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087801436","display_name":"Pawan Sasanka Ammanamanchi","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Pawan Sasanka Ammanamanchi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059265033","display_name":"Dirk Groeneveld","orcid":"https://orcid.org/0000-0002-8274-768X","observed_orcids":["https://orcid.org/0000-0002-8274-768X"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Dirk Groeneveld","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ethan Smith","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Ethan Smith","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Eric Tang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Eric Tang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":916,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8140000104904175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8140000104904175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09160000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.02319999970495701,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/language-model-evaluation","display_name":"language model evaluation","score":0.7710000276565552},{"id":"https://openalex.org/keywords/benchmark-suite","display_name":"benchmark suite","score":0.578000009059906},{"id":"https://openalex.org/keywords/onnx-runtime","display_name":"ONNX Runtime","score":0.4090000092983246},{"id":"https://openalex.org/keywords/rulers","display_name":"rulers","score":0.4580000042915344},{"id":"https://openalex.org/keywords/hugging-face-datasets","display_name":"HuggingFace Datasets","score":0.4309999942779541},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.4650000035762787},{"id":"https://openalex.org/keywords/benchmarking-metrics","display_name":"benchmarking metrics","score":0.34200000762939453},{"id":"https://openalex.org/keywords/benchmarking-tool","display_name":"benchmarking tool","score":0.38600000739097595}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7005000114440918},{"id":"https://openalex.org/C2776654903","wikidata":"https://www.wikidata.org/wiki/Q2601463","display_name":"SAFER","level":2,"score":0.5645999908447266},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5576000213623047},{"id":"https://openalex.org/C2778689934","wikidata":"https://www.wikidata.org/wiki/Q1313396","display_name":"Headline","level":2,"score":0.5069000124931335},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.43810001015663147},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.3785000145435333},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.36169999837875366},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.3562000095844269},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3325999975204468},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.32749998569488525},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.31040000915527344},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3070000112056732},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.25360000133514404},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.2535000145435333}],"mesh":[],"locations_count":25,"locations":[{"id":"pmh:oai:zenodo.org:5371629","doi":null,"is_oa":true,"landing_page_url":"https://zenodo.org/record/5371629","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"software"},{"id":"pmh:oai:zenodo.org:6332975","doi":null,"is_oa":true,"landing_page_url":"https://zenodo.org/record/6332975","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"software"},{"id":"pmh:oai:zenodo.org:7413426","doi":null,"is_oa":true,"landing_page_url":"https://zenodo.org/record/7413426","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"software"},{"id":"doi:10.5281/zenodo.10256836","doi":"https://doi.org/10.5281/zenodo.10256836","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.10256836","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.10600400","doi":"https://doi.org/10.5281/zenodo.10600400","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.10600400","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.10829972","doi":"https://doi.org/10.5281/zenodo.10829972","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.10829972","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.12608602","doi":"https://doi.org/10.5281/zenodo.12608602","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.12608602","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.13694023","doi":"https://doi.org/10.5281/zenodo.13694023","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.13694023","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.13905736","doi":"https://doi.org/10.5281/zenodo.13905736","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.13905736","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.14216804","doi":"https://doi.org/10.5281/zenodo.14216804","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.14216804","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.14506035","doi":"https://doi.org/10.5281/zenodo.14506035","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.14506035","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.14970487","doi":"https://doi.org/10.5281/zenodo.14970487","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.14970487","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.15699229","doi":"https://doi.org/10.5281/zenodo.15699229","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.15699229","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.16737642","doi":"https://doi.org/10.5281/zenodo.16737642","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.16737642","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.17728786","doi":"https://doi.org/10.5281/zenodo.17728786","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17728786","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.18393948","doi":"https://doi.org/10.5281/zenodo.18393948","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.18393948","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.18394108","doi":"https://doi.org/10.5281/zenodo.18394108","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.18394108","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.18636344","doi":"https://doi.org/10.5281/zenodo.18636344","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.18636344","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.20122284","doi":"https://doi.org/10.5281/zenodo.20122284","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.20122284","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.22214906","doi":"https://doi.org/10.5281/zenodo.22214906","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22214906","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.22214970","doi":"https://doi.org/10.5281/zenodo.22214970","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.22214970","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.5371628","doi":"https://doi.org/10.5281/zenodo.5371628","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.5371628","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.5371629","doi":"https://doi.org/10.5281/zenodo.5371629","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.5371629","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.6332975","doi":"https://doi.org/10.5281/zenodo.6332975","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.6332975","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.7413426","doi":"https://doi.org/10.5281/zenodo.7413426","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.7413426","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"}],"best_oa_location":{"id":"pmh:oai:zenodo.org:5371629","doi":null,"is_oa":true,"landing_page_url":"https://zenodo.org/record/5371629","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"software"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W6891680161","https://openalex.org/W4285107336","https://openalex.org/W7126263866","https://openalex.org/W3201734921","https://openalex.org/W6891713639","https://openalex.org/W3173777717","https://openalex.org/W6968372622","https://openalex.org/W7116184612","https://openalex.org/W4385565879"],"abstract_inverted_index":{"v0.4.13":[0],"Release":[1],"Notes":[2,1077],"A":[3,956,992],"fix-focused":[4],"release.":[5],"The":[6,61],"main":[7],"fixes":[8,1131],"are":[9,304,494,836,922,935],"for":[10,34,38,98,244,924,976,1081,1197,1211,1314],"few-shot":[11,59,88,216,754],"leakage,":[12],"a":[13,115,119,348,465,563,595,632,683,766,848,945,960,996,1030,1084],"multiple-choice":[14,424],"filter":[15,734],"bug,":[16],"and":[17,25,77,102,130,214,224,230,341,379,446,456,461,491,497,530,652,666,724,809,868,1029,1042,1053,1091,1114,1176,1214],"group":[18,350,524],"stderr,":[19],"alongside":[20],"two":[21],"new":[22,27,368],"ONNX":[23,247,256],"backends":[24],"eight":[26],"benchmark":[28,394,426],"suites.":[29],"Also":[30],"updated":[31],"most":[32],"configs":[33,319],"datasets>=4,":[35],"which":[36],"accounts":[37],"much":[39],"of":[40,125,265,1174],"the":[41,65,82,87,92,152,158,192,208,215,233,290,372,380,489,518,548,569,589,601,607,678,727,753,773,815,826,889,948,966,1001,1004,1054,1093,1097,1129,1166],"diff":[42],"by":[43,73,104,141,177,222,249,267,274,320,351,362,389,400,407,416,427,437,448,459,473,499,513,526,538,552,572,610,624,638,654,668,685,699,712,731,743,760,779,792,805,818,829,839,853,864,880,892,900,916,928,939,952,969,979,988,1007,1019,1036,1045,1058,1072,1185,1192,1202,1217,1229,1243,1253,1264,1271,1282,1295,1307],"volume.":[44],"Highlights":[45],"Bug":[46],"Fixes":[47,48],"that":[49,117,870,1179],"may":[50,1109],"shift":[51],"previously":[52,1095],"reported":[53,151],"numbers:":[54],"Eval":[55,325],"documents":[56,333],"leaked":[57],"into":[58,69,95,484,1100],"prompts.":[60],"sampler":[62,1094],"could":[63],"draw":[64],"document":[66,1099],"under":[67,599,677],"test":[68],"its":[70,1034,1101],"own":[71,1102],"demonstrations":[72],"@SiavashShams":[74,830],"in":[75,106,143,179,251,269,276,322,353,364,402,409,418,429,439,450,475,501,515,528,540,554,574,612,626,640,656,670,687,701,708,714,745,762,781,794,802,807,820,831,841,855,866,882,894,902,912,918,930,941,954,971,981,990,1009,1021,1038,1047,1060,1074,1133,1178,1187,1194,1204,1219,1224,1231,1245,1255,1266,1273,1284,1288,1297,1309],"#3978,":[76],"gen_prefix":[78,1086],"was":[79,161,675],"resolved":[80,605],"against":[81],"eval":[83,571,1098],"doc":[84,89],"rather":[85],"than":[86],"\u2014":[90,136,240,254,294,302,310,326,358,367,392,412,422,433,443,453,464,504,543,558,577,615,629,647,659,673,690,704,717,735,748,765,1026,1140],"splicing":[91],"evaluated":[93],"question":[94],"every":[96,578],"shot":[97],"RULER":[99,576],"niah_single_1,":[100,1088],"humaneval_instruct,":[101,1089],"humaneval_64_instruct":[103],"@adityasingh2400":[105],"#3979":[107],"MultiChoiceRegexFilter":[108],"prefix-shadowing.":[109],"Regex":[110],"alternation":[111],"is":[112,219,603,719,799,845,858,944,950,999,1119,1262],"leftmost-wins,":[113],"so":[114,562],"choice":[116,121,1302],"prefixed":[118],"longer":[120,187,196,567,617,750,812,964,1014],"(\"Guilty\"":[122],"vs":[123],"\"Guilty":[124],"Romance\")":[126],"matched":[127],"inside":[128],"it":[129,266,887],"scored":[131],"correct":[132,212],"answers":[133],"as":[134,343,592,777],"wrong":[135,679],"visible":[137],"on":[138,263,814,1016,1106,1172],"BBH":[139],"movie_recommendation":[140],"@iamsharduld":[142,178,1073],"#3884":[144],"Group":[145],"stderr":[146,155],"with":[147,260,279,581,1239],"weight_by_size:":[148],"false.":[149],"Groups":[150],"size-weighted":[153,349],"pooled":[154],"even":[156],"when":[157,588,620,752,886,937,947,1259],"point":[159],"estimate":[160],"an":[162,204,873],"unweighted":[163],"mean,":[164],"giving":[165],"error":[166],"bars":[167],"up":[168,232],"to":[169,207,467,535,861,872,878,914,1067,1122,1149],"~3x":[170],"too":[171],"narrow.":[172],"Only":[173],"unequal-sized":[174],"subtasks":[175],"change":[176],"#3882":[180],"minerva_math":[181,1113,1134],"answer":[182,205,1301],"normalization.":[183],"sqrt":[184],"shorthand":[185],"no":[186,195,566,616,749,811,963,1013],"corrupts":[188],"indexed":[189],"roots":[190],"(#4037),":[191],"thousands-separator":[193],"strip":[194,1207],"fuses":[197],"digit":[198],"tuples":[199],"(0,1":[200],"\u2192":[201],"01,":[202],"#4039),":[203],"identical":[206],"gold":[209],"now":[210,495,546,604,720,769,789,1116,1170],"scores":[211],"(#4034),":[213],"prompt":[217],"LaTeX":[218],"corrected":[220,707,1052],"(#4045)":[221],"@feiiiiii5":[223,929,953],"@nata2627.":[225],"putnam_axiom":[226],"shares":[227],"these":[228,303],"helpers":[229],"picks":[231],"same":[234,1002,1167],"fixes.":[235],"New":[236,307],"Model":[237,245,783],"Backends":[238,784],"onnxruntime":[239,242],"raw":[241],"backend":[243,817],"Builder":[246],"exports":[248],"@amd-sourjya":[250],"#3984":[252],"onnxruntime-genai":[253,1175],"cross-platform":[255],"Runtime":[257],"GenAI":[258],"backend,":[259],"winml":[261,1169],"refactored":[262],"top":[264,1173],"@thiagocrepaldi":[268],"#3960":[270],"Megatron-LM":[271],"v0.18":[272],"compatibility":[273],"@oberpierre":[275],"#3991":[277],"Install":[278],"pip":[280],"install":[281,289],"lm_eval[onnxruntime]":[282],"or":[283,298],"lm_eval[onnxruntime-genai].":[284],"For":[285],"non-CPU":[286],"execution":[287],"providers":[288],"matching":[291],"wheel":[292],"instead":[293],"onnxruntime-gpu":[295],"/":[296,300,507,509,521,643,645],"onnxruntime-rocm,":[297],"onnxruntime-genai-cuda":[299],"onnxruntime-genai-directml":[301],"mutually":[305],"exclusive.":[306],"Tasks":[308],"LongProc":[309],"long-context":[311],"procedural":[312],"reasoning,":[313],"6":[314],"task":[315,511,579,680,697,710,728,968,1137,1158,1227,1291],"types":[316,1065],"across":[317],"16":[318],"@xiye17":[321],"#3544":[323],"Uncheatable":[324],"contamination-resistant":[327],"evaluation:":[328],"rolling":[329],"log-likelihood":[330],"over":[331,772],"recently-published":[332],"from":[334,650,694,847,995,1209],"Wikipedia,":[335],"GitHub,":[336],"BBC":[337],"News,":[338],"arXiv,":[339],"bioRxiv,":[340],"AO3,":[342],"15":[344],"category":[345],"tasks":[346,978,1082,1108,1152,1216],"plus":[347],"@ziqing-huang":[352],"#3442":[354],"TyDiQA":[355],"Gold":[356],"Passage":[357],"9-language":[359],"multilingual":[360],"QA":[361],"@bongho":[363,390,428,449,1283],"#4044":[365],"LegalBench":[366,374,1278],"suite,":[369],"19":[370],"tasks:":[371],"HELM-lite":[373,1277],"subset":[375,1279],"(5":[376,1280],"tasks,":[377,387],"#3860)":[378],"Contract":[381],"NLI":[382],"suite":[383],"(14":[384],"NDA":[385],"entailment":[386],"#3954)":[388],"IndicParam":[391],"MCQ":[393],"covering":[395],"12":[396],"low-resource":[397],"Indic":[398],"languages":[399],"@hjoshifonteva":[401],"#3826":[403],"IndicXNLI":[404],"Gujarati":[405],"(indicxnli_gu)":[406],"@bhaumik611":[408],"#4056":[410],"GreekMMLU":[411],"official":[413],"native-sourced":[414],"configuration":[415],"@mersinkonomi":[417],"#3581":[419],"Physics":[420],"GRE":[421],"InflectionAI":[423],"physics":[425],"#3853":[430],"Putnam":[431],"Axiom":[432],"competition-level":[434],"mathematical":[435],"reasoning":[436],"@baberabb":[438,460,500,917,1186,1193],"#3998":[440],"Portuguese":[441],"Bench":[442,557],"ASSIN2":[444],"RTE":[445],"STS":[447],"#3812":[451],"Catalanbench":[452],"cieaCOVA":[454],"(#4043)":[455],"Terretaqa":[457],"(#3784)":[458],"@ivanmartinezmurillo":[462],"MLQA":[463],"tag":[466],"run":[468],"all":[469],"variants":[470],"at":[471,1161],"once":[472],"@ivanbaldo":[474],"#2977":[476],"Task":[477,1142],"Changes":[478,1143],"Correctness":[479],"&":[480,785,906],"Prompts":[481],"global_piqa":[482,1136],"restructured":[483],"parallel/non-parallel":[485],"\u00d7":[486],"cloze/generation":[487],"variants;":[488],"global_piqa_completions":[490],"global_piqa_prompted":[492],"groups":[493],"global_piqa_cloze":[496],"global_piqa_generation":[498],"#3816":[502],"IrokoBench":[503],"repaired":[505,1025],"afrimmlu":[506],"afrimgsm":[508],"afrixnli":[510,532],"registration":[512],"@discobot":[514,553,1308],"#3841,":[516],"fixed":[517,899,1057],"broken":[519],"afrisenti":[520],"mafand":[522],"prompt_2":[523],"references":[525],"@DaoyuanLi2816":[527,700,713,1230,1296],"#3847,":[529],"switched":[531],"prompt_1":[533],"doc_to_text":[534],"Jinja":[536],"braces":[537],"@Solaris-star":[539],"#3944":[541],"KorMedMCQA":[542],"answer-choice":[544],"extraction":[545,637],"follows":[547],"paper's":[549],"Appendix":[550,1305],"B":[551,1306],"#3842":[555],"JSONSchema":[556],"per-sample":[559],"validation":[560],"timeout":[561,828],"pathological":[564],"schema":[565],"hangs":[568],"whole":[570],"@hancheolcho":[573],"#3923":[575],"crashed":[580],"TypeError:":[582],"unhashable":[583],"type:":[584],"'dict'":[585],"before":[586,606,636,663],"inference":[587],"tokenizer":[590],"arrived":[591],"anything":[593],"but":[594],"plain":[596],"string":[597,875,983],"(e.g.":[598],"local-chat-completions);":[600],"name":[602,729,962],"cached":[608],"lookup":[609],"@nata2627":[611],"#4048":[613],"NIAH":[614],"loops":[618],"forever":[619],"max_seq_lengths":[621],"<":[622],"4096":[623],"@vnayakde":[625],"#3372":[627],"LongBench":[628],"code_sim_score":[630],"skips":[631],"blank":[633],"leading":[634],"line":[635],"@cameronshinn":[639],"#3921":[641],"FDA":[642],"SWDE":[644],"SQuAD_completion":[646,1215],"whitespace":[648,1208],"stripped":[649],"input":[651],"target":[653],"@EphraiemSarabamoun":[655,1218],"#3795":[657],"med_prescriptions":[658],"requires":[660],"both":[661],"keys":[662,921],"combining":[664],"complaints":[665],"diagnosis":[667],"@Anai-Guo":[669,865,1037,1244],"#4041":[671],"HumanEval":[672],"humaneval_random_span_infilling_light":[674],"registered":[676,967],"name,":[681],"causing":[682],"KeyError":[684],"@jaydeepborkar":[686,1203],"#3768":[688],"INCLUDE":[689],"stray":[691],"space":[692,1287],"removed":[693],"North":[695,1289],"Macedonian":[696,1290],"identifiers":[698,1292],"#3848":[702],"DarijaBench":[703,1225],"trasnlation":[705],"typo":[706,1223],"translation":[709,1226],"names":[711,1138,1228],"#3824":[715],"Unitxt":[716],"dataset_kwargs":[718],"passed":[721,859],"through":[722,860],"(#3230)":[723],"init":[725],"sets":[726],"(#3225)":[730],"@mprahl":[732],"format_span":[733],"normalizes":[736],"labels":[737],"only,":[738],"leaving":[739],"entity":[740],"text":[741],"untouched":[742],"@k-dickinson":[744],"#3887":[746],"ContextSampler":[747],"crashes":[751],"pool":[755],"contains":[756],"duplicate":[757],"eval-doc":[758],"rows":[759],"@Kymi808":[761],"#3790":[763],"fewshot_config.split":[764,768],"nested":[767,849],"takes":[770],"precedence":[771],"inherited":[774],"top-level":[775],"fewshot_split,":[776],"documented":[778],"@chuenchen309":[780],"#3937":[782],"APIs":[786],"Chat":[787],"completions":[788],"support":[790],"think_end_token":[791],"@yaodong-shen":[793],"#3959":[795],"auto:N":[796],"batch":[797],"size":[798],"parsed":[800],"correctly":[801,987],"API":[803,823],"models":[804],"@AbdullahRasheed45":[806,819],"#3970,":[808],"batch_size=\"auto\"":[810],"raises":[813],"neuronx":[816],"#3971":[821],"Synchronous":[822],"requests":[824,927],"honor":[825],"configured":[827],"#3995":[832],"Anthropic":[833,1249],"stop":[834,1250],"sequences":[835,1251],"kept":[837],"non-empty":[838],"@he-yufeng":[840,1254],"#3822":[842],"HF:":[843],"max_length":[844],"detected":[846],"text_config":[850],"(Gemma3":[851],"multimodal)":[852],"@OrionArchitekton":[854],"#3916,":[856],"max_cpu_memory":[857],"accelerate's":[862],"max_memory":[863],"#4016,":[867],"tokenizers":[869],"decode":[871],"empty":[874],"fall":[876],"back":[877],"tokenizer.eos_token":[879],"@ganeshr10":[881],"#3657":[883],"vLLM":[884],"warns":[885],"ignores":[888],"device":[890,1260],"argument":[891,897,1261],"@Apeironics":[893,1265],"#3803;":[895],"sglang":[896,1269],"passing":[898],"@wm901115nwpu":[901,1272],"#3817":[903],"Core,":[904],"CLI":[905,1005],"Config":[907],"--metadata":[908],"accepts":[909],"key=value":[910],"pairs":[911],"addition":[913],"JSON":[915],"#4054":[919],"--use_cache":[920],"hashable":[923],"multimodal":[925],"image/byte":[926],"#4040":[931],"Cache":[932],"parent":[933],"directories":[934],"created":[936],"missing":[938],"@sdivyanshu90":[940],"#4047;":[942],"delete_cache":[943],"no-op":[946],"directory":[949,958],"absent":[951],"#4035":[955],"local":[957],"sharing":[959],"task's":[961],"shadows":[965],"@nloughl":[970],"#3670":[972],"Function":[973],"resolving":[974],"works":[975],"custom":[977],"@SkyR0ver":[980],"#3992;":[982],"dictionary":[984],"arguments":[985],"parse":[986],"@tandede":[989],"#4020":[991],"scalar":[993,1064],"seed":[994],"config":[997],"file":[998],"normalized":[1000],"way":[1003],"does":[1006],"@winklemad":[1008],"#4003":[1010],"Sample":[1011],"counts":[1012],"depend":[1015],"metric":[1017,1043,1050],"ordering":[1018],"@arthi-arumugam-git":[1020],"#4068":[1022],"scripts/requests_caching.py":[1023],"entrypoint":[1024],"bad":[1027],"kwargs":[1028],"helper":[1031],"defined":[1032],"after":[1033],"use":[1035],"#4059":[1039],"chrF++":[1040],"aggregation":[1041],"(word_order=2)":[1044],"@KrishVenky":[1046],"#3780;":[1048],"TER":[1049],"direction":[1051],"chrF":[1055],"docstring":[1056],"@borgr":[1059],"#3993":[1061],"All":[1062],"numpy":[1063],"serialize":[1066],"JSON,":[1068],"not":[1069,1110],"just":[1070],"int64/int32,":[1071],"#3885":[1075],"Migration":[1076],"Few-shot":[1078],"prompts":[1079],"changed":[1080,1139],"using":[1083],"document-specific":[1085],"(RULER":[1087],"humaneval_64_instruct)":[1090],"anywhere":[1092],"drew":[1096],"shots.":[1103],"Prior":[1104],"numbers":[1105],"those":[1107],"be":[1111],"comparable.":[1112],"leaderboard_math_*":[1115],"diverge.":[1117],"lm_eval/tasks/leaderboard/math/utils.py":[1118],"deliberately":[1120],"frozen":[1121],"reproduce":[1123],"Open":[1124],"LLM":[1125],"Leaderboard":[1126],"v2":[1127],"scoring;":[1128],"normalization":[1130],"landed":[1132],"only.":[1135],"see":[1141],"above.":[1144],"If":[1145],"you":[1146,1154],"pinned":[1147,1240],"datasets<4":[1148],"keep":[1150,1248],"script-based":[1151,1162],"working,":[1153],"can":[1155],"unpin.":[1156],"Out-of-tree":[1157],"YAMLs":[1159],"pointing":[1160],"datasets":[1163],"still":[1164],"need":[1165],"treatment.":[1168],"sits":[1171],"pulls":[1177],"extra.":[1180],"What's":[1181],"Changed":[1182],"feat:":[1183],"0.4.13.dev0":[1184],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3764":[1188],"update":[1189],"global":[1190],"piqa":[1191],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3816":[1195],"fix":[1196,1268],"keyerror":[1198],"while":[1199],"running":[1200],"humaneval_infilling":[1201],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3768":[1205],"fix:":[1206,1247],"input/target":[1210],"FDA,":[1212],"SWDE,":[1213],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3795":[1220],"Fix":[1221,1286,1311],"\"trasnlation\"":[1222],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3824":[1232],"fix(ruler):":[1233],"replace":[1234],"defunct":[1235],"hotpot":[1236],"dataset":[1237,1312],"host":[1238],"HF":[1241],"mirror":[1242],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3806":[1246],"nonempty":[1252],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3822":[1256],"fix(vllm):":[1257],"warn":[1258],"ignored":[1263],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3803":[1267],"args":[1270],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3817":[1274],"feat(legalbench):":[1275],"add":[1276],"tasks)":[1281],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3860":[1285],"(INCLUDE":[1293],"suite)":[1294],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3848":[1298],"fix(kormedmcqa):":[1299],"extract":[1300],"per":[1303],"paper":[1304],"https://github.com/EleutherAI/lm-evaluation-harness/pull/3842":[1310],"paths":[1313]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W6893809404","counts_by_year":[{"year":2026,"cited_by_count":261},{"year":2025,"cited_by_count":246},{"year":2024,"cited_by_count":231},{"year":2023,"cited_by_count":156},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":3}],"updated_date":"2026-11-20T13:11:24.684918","created_date":"2025-10-10T00:00:00"}
