{"id":"https://openalex.org/W4385570331","doi":"https://doi.org/10.18653/v1/2023.findings-acl.540","title":"Aligning Offline Metrics and Human Judgments of Value for Code Generation Models","display_name":"Aligning Offline Metrics and Human Judgments of Value for Code Generation Models","alternate_titles":[],"publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4385570331","doi":"https://doi.org/10.18653/v1/2023.findings-acl.540"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-acl.540","doi":"https://doi.org/10.18653/v1/2023.findings-acl.540","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-acl.540","pdf_url":"https://aclanthology.org/2023.findings-acl.540.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2023","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4385570331"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.findings-acl.540.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047676499","display_name":"Victor C. Dibia","orcid":"https://orcid.org/0000-0002-1839-5632","observed_orcids":["https://orcid.org/0000-0002-1839-5632"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Victor Dibia","raw_affiliation_strings":["Microsoft Research, Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071293344","display_name":"Adam Fourney","orcid":"https://orcid.org/0000-0002-4986-7794","observed_orcids":["https://orcid.org/0000-0002-4986-7794"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adam Fourney","raw_affiliation_strings":["Microsoft Research, Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Gagan Bansal","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gagan Bansal","raw_affiliation_strings":["Microsoft Research, Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050672404","display_name":"Forough Poursabzi-Sangdeh","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Forough Poursabzi-Sangdeh","raw_affiliation_strings":["Microsoft Research, Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100349055","display_name":"Han Liu","orcid":"https://orcid.org/0000-0002-7731-8258","observed_orcids":["https://orcid.org/0000-0002-7731-8258"]},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Han Liu","raw_affiliation_strings":["University of Chicago, Chicago, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Chicago, Chicago, United States","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057541630","display_name":"Saleema Amershi","orcid":"https://orcid.org/0000-0002-3294-7288","observed_orcids":["https://orcid.org/0000-0002-3294-7288"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saleema Amershi","raw_affiliation_strings":["Microsoft Research, Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.9227,"has_fulltext":true,"cited_by_count":37,"citation_normalized_percentile":{"value":0.96762204,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"8516","last_page":"8528"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.0015999999595806003,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.8140000104904175},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9610000252723694},{"id":"https://openalex.org/keywords/code-quality-metrics","display_name":"code quality metrics","score":0.5889999866485596},{"id":"https://openalex.org/keywords/functional-correctness","display_name":"functional correctness","score":0.871999979019165},{"id":"https://openalex.org/keywords/unit-testing","display_name":"unit testing","score":0.8199999928474426},{"id":"https://openalex.org/keywords/syntactic-similarity","display_name":"syntactic similarity","score":0.800000011920929},{"id":"https://openalex.org/keywords/productivity-improvement","display_name":"productivity improvement","score":0.5049999952316284},{"id":"https://openalex.org/keywords/user-study","display_name":"user study","score":0.46399998664855957}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.9369077682495117},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8510088920593262},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5378263592720032},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4993741512298584},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.48906761407852173},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.44834867119789124},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.4432539641857147},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.41251927614212036},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.20604872703552246},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.findings-acl.540","doi":"https://doi.org/10.18653/v1/2023.findings-acl.540","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-acl.540","pdf_url":"https://aclanthology.org/2023.findings-acl.540.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2210.16494","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2210.16494","pdf_url":"https://arxiv.org/pdf/2210.16494","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2210.16494","doi":"https://doi.org/10.48550/arxiv.2210.16494","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2210.16494","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-acl.540","doi":"https://doi.org/10.18653/v1/2023.findings-acl.540","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-acl.540","pdf_url":"https://aclanthology.org/2023.findings-acl.540.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385570331.pdf","grobid_xml":"https://content.openalex.org/works/W4385570331.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1970859802","https://openalex.org/W1972141422","https://openalex.org/W1973682096","https://openalex.org/W2074032109","https://openalex.org/W2101105183","https://openalex.org/W2140609933","https://openalex.org/W2154652894","https://openalex.org/W2728773317","https://openalex.org/W2754629507","https://openalex.org/W2903709763","https://openalex.org/W2950621612","https://openalex.org/W2954451301","https://openalex.org/W2963935794","https://openalex.org/W2995006168","https://openalex.org/W3025993830","https://openalex.org/W3034269545","https://openalex.org/W3089307846","https://openalex.org/W3126675481","https://openalex.org/W3135708166","https://openalex.org/W3148399464","https://openalex.org/W3170092793","https://openalex.org/W3177765786","https://openalex.org/W3177813494","https://openalex.org/W3195140297","https://openalex.org/W3198685994","https://openalex.org/W3213368902","https://openalex.org/W4224060952","https://openalex.org/W4224308101","https://openalex.org/W4226287673","https://openalex.org/W4226485558","https://openalex.org/W4252398070","https://openalex.org/W4280605339","https://openalex.org/W4281669078","https://openalex.org/W4281763794","https://openalex.org/W4283156811","https://openalex.org/W4287077924","https://openalex.org/W4307654398","https://openalex.org/W7151510318"],"related_works":["https://openalex.org/W4385573348","https://openalex.org/W4375947047","https://openalex.org/W7172498920","https://openalex.org/W4393969224","https://openalex.org/W7126383469","https://openalex.org/W7169845512","https://openalex.org/W4375859932","https://openalex.org/W4402516191","https://openalex.org/W4402703807","https://openalex.org/W4402835407"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,54],"have":[3],"demonstrated":[4],"great":[5],"potential":[6],"to":[7,91],"assist":[8],"programmers":[9,74],"in":[10,29],"generating":[11],"code.For":[12],"such":[13],"human-AI":[14],"pair":[15],"programming":[16],"scenarios,":[17],"we":[18,66,96],"empirically":[19],"demonstrate":[20],"that":[21,68,78,101,110],"while":[22,69],"generated":[23],"code":[24,77],"are":[25],"most":[26],"often":[27],"evaluated":[28],"terms":[30],"of":[31],"their":[32],"functional":[33,103],"correctness":[34,42,70,104],"(i.e.,":[35],"whether":[36],"generations":[37],"pass":[38],"available":[39],"unit":[40,80],"tests),":[41],"does":[43],"not":[44],"fully":[45],"capture":[46],"(e.g.,":[47],"may":[48,55],"underestimate)":[49],"the":[50,87],"productivity":[51],"gains":[52,125],"these":[53],"provide.Through":[56],"a":[57,93,98,113],"user":[58],"study":[59],"with":[60,117],"N":[61],"=":[62],"49":[63],"experienced":[64],"programmers,":[65],"show":[67,109],"captures":[71],"high-value":[72],"generations,":[73],"still":[75],"rate":[76],"fails":[79],"tests":[81],"as":[82],"valuable":[83],"if":[84],"it":[85,111],"reduces":[86],"overall":[88],"effort":[89],"needed":[90],"complete":[92],"coding":[94],"task.Finally,":[95],"propose":[97],"hybrid":[99],"metric":[100],"combines":[102],"and":[105,108,119,128],"syntactic":[106],"similarity":[107],"achieves":[112],"14%":[114],"stronger":[115],"correlation":[116],"value":[118],"can":[120],"therefore":[121],"better":[122],"represent":[123],"real-world":[124],"when":[126],"evaluating":[127],"comparing":[129],"models.":[130]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4385570331","counts_by_year":[{"year":2026,"cited_by_count":9},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2023-08-05T00:00:00"}
