{"id":"https://openalex.org/W4300989059","doi":"https://doi.org/10.48550/arxiv.2208.08227","title":"MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation","display_name":"MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation","publication_year":2022,"publication_date":"2022-08-17","ids":{"openalex":"https://openalex.org/W4300989059","doi":"https://doi.org/10.48550/arxiv.2208.08227"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2208.08227","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.08227","pdf_url":"https://arxiv.org/pdf/2208.08227","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2208.08227","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061118771","display_name":"Federico Cassano","orcid":"https://orcid.org/0000-0002-9318-7454"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cassano, Federico","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002038171","display_name":"John Gouwar","orcid":"https://orcid.org/0000-0003-0494-7245"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gouwar, John","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101711261","display_name":"Daniel Nguyen","orcid":"https://orcid.org/0000-0002-4609-0844"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027365771","display_name":"Sydney Nguyen","orcid":"https://orcid.org/0000-0002-7053-699X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Sydney","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085474894","display_name":"Luna Phipps-Costin","orcid":"https://orcid.org/0009-0001-4398-6987"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Phipps-Costin, Luna","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021568199","display_name":"Donald Pinckney","orcid":"https://orcid.org/0000-0001-8612-5178"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pinckney, Donald","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054166496","display_name":"Ming Ho Yee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yee, Ming-Ho","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018675797","display_name":"Yangtian Zi","orcid":"https://orcid.org/0000-0003-2606-3280"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zi, Yangtian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075492610","display_name":"Carolyn Jane Anderson","orcid":"https://orcid.org/0000-0001-5717-4210"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anderson, Carolyn Jane","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010858670","display_name":"Molly Q Feldman","orcid":"https://orcid.org/0000-0002-5222-7720"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feldman, Molly Q","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044042294","display_name":"Arjun Guha","orcid":"https://orcid.org/0000-0002-7493-3271"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guha, Arjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075185353","display_name":"Michael Greenberg","orcid":"https://orcid.org/0000-0003-0014-7670"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Greenberg, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5034240415","display_name":"Abhinav Jangda","orcid":"https://orcid.org/0000-0002-4849-6776"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jangda, Abhinav","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":13,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9718000292778015,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9519000053405762,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8457620143890381},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.7392151355743408},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.7380765676498413},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.6241380572319031},{"id":"https://openalex.org/keywords/programming-paradigm","display_name":"Programming paradigm","score":0.5117030143737793},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5044976472854614},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.46020621061325073},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3457828164100647},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.08192402124404907}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8457620143890381},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.7392151355743408},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.7380765676498413},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6241380572319031},{"id":"https://openalex.org/C34165917","wikidata":"https://www.wikidata.org/wiki/Q188267","display_name":"Programming paradigm","level":2,"score":0.5117030143737793},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5044976472854614},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.46020621061325073},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3457828164100647},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.08192402124404907},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2208.08227","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.08227","pdf_url":"https://arxiv.org/pdf/2208.08227","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2208.08227","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2208.08227","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2208.08227","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.08227","pdf_url":"https://arxiv.org/pdf/2208.08227","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.75,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2341492732","https://openalex.org/W3187193180","https://openalex.org/W106542691","https://openalex.org/W4287027380","https://openalex.org/W1699080303","https://openalex.org/W4297799326","https://openalex.org/W3116064965","https://openalex.org/W2378211422","https://openalex.org/W3193760048","https://openalex.org/W2517841089"],"abstract_inverted_index":{"Large":[0],"language":[1,11,14,34,171,174],"models":[2,17,29,41],"have":[3],"demonstrated":[4],"the":[5,20,74,102,127,168,180],"ability":[6],"to":[7,35,69,84,92,100,108,166,188,200],"generate":[8,43],"both":[9,193],"natural":[10],"and":[12,105,118,138,173,195,205],"programming":[13,95,116,159,190],"text.":[15],"Such":[16],"open":[18],"up":[19],"possibility":[21],"of":[22,115,130,158,170,183],"multi-language":[23,128],"code":[24,27,39,66,78,89,133,185],"generation:":[25],"could":[26],"generation":[28,40,67,79,90,134,186],"generalize":[30],"knowledge":[31],"from":[32],"one":[33],"another?":[36],"Although":[37],"contemporary":[38],"can":[42],"semantically":[44],"correct":[45],"Python":[46,88,151],"code,":[47],"little":[48],"is":[49,192],"known":[50],"about":[51],"their":[52],"abilities":[53],"with":[54],"other":[55,154],"languages.":[56,71,96,155,206],"We":[57,72,97,140],"propose":[58],"MultiPL-E,":[59],"a":[60,113],"system":[61],"for":[62,152],"translating":[63],"unit":[64],"test-driven":[65],"benchmarks":[68,91,187],"new":[70,122,189,202],"create":[73],"first":[75],"massively":[76],"multilingual":[77],"benchmark":[80,104,107],"by":[81],"using":[82],"MultiPL-E":[83,99,163,181],"translate":[85],"two":[86],"popular":[87],"18":[93,109],"additional":[94],"use":[98],"extend":[101],"HumanEval":[103],"MBPP":[106],"languages":[110,160,191],"that":[111,142],"encompass":[112],"range":[114,157],"paradigms":[117],"popularity.":[119],"Using":[120],"these":[121],"parallel":[123],"benchmarks,":[124,204],"we":[125],"evaluate":[126,201],"performance":[129,149],"three":[131],"state-of-the-art":[132],"models:":[135],"Codex,":[136],"CodeGen,":[137],"InCoder.":[139],"find":[141],"Codex":[143],"matches":[144],"or":[145],"even":[146],"exceeds":[147],"its":[148],"on":[150,176],"several":[153],"The":[156],"represented":[161],"in":[162],"allow":[164],"us":[165],"explore":[167],"impact":[169],"frequency":[172],"features":[175],"model":[177],"performance.":[178],"Finally,":[179],"approach":[182],"compiling":[184],"scalable":[194],"extensible,":[196],"making":[197],"it":[198],"straightforward":[199],"models,":[203]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":6}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}