{"id":"https://openalex.org/W4312971267","doi":"https://doi.org/10.1145/3510454.3528648","title":"CrystalBLEU","display_name":"CrystalBLEU","publication_year":2022,"publication_date":"2022-05-21","ids":{"openalex":"https://openalex.org/W4312971267","doi":"https://doi.org/10.1145/3510454.3528648"},"language":"en","primary_location":{"id":"doi:10.1145/3510454.3528648","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3510454.3528648","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM/IEEE 44th International Conference on Software Engineering: Companion Proceedings","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002459483","display_name":"Aryaz Eghbali","orcid":"https://orcid.org/0000-0001-9763-8147"},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Aryaz Eghbali","raw_affiliation_strings":["University of Stuttgart, Stuttgart, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Stuttgart, Stuttgart, Germany","institution_ids":["https://openalex.org/I100066346"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013438083","display_name":"Michael Pradel","orcid":"https://orcid.org/0000-0003-1623-498X"},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Michael Pradel","raw_affiliation_strings":["University of Stuttgart, Stuttgart, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Stuttgart, Stuttgart, Germany","institution_ids":["https://openalex.org/I100066346"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I100066346"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"341","last_page":"342"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8760180473327637},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6027196645736694},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.5860350131988525},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5734845399856567},{"id":"https://openalex.org/keywords/bleu","display_name":"BLEU","score":0.5601041316986084},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.5589628219604492},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5502252578735352},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5473761558532715},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.5394248962402344},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5357477068901062},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.4748285710811615},{"id":"https://openalex.org/keywords/abstract-syntax-tree","display_name":"Abstract syntax tree","score":0.4473978281021118},{"id":"https://openalex.org/keywords/abstract-syntax","display_name":"Abstract syntax","score":0.41047194600105286},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.28686290979385376}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8760180473327637},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6027196645736694},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.5860350131988525},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5734845399856567},{"id":"https://openalex.org/C622187","wikidata":"https://www.wikidata.org/wiki/Q3500773","display_name":"BLEU","level":3,"score":0.5601041316986084},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.5589628219604492},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5502252578735352},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5473761558532715},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.5394248962402344},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5357477068901062},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4748285710811615},{"id":"https://openalex.org/C58646249","wikidata":"https://www.wikidata.org/wiki/Q127380","display_name":"Abstract syntax tree","level":3,"score":0.4473978281021118},{"id":"https://openalex.org/C114408938","wikidata":"https://www.wikidata.org/wiki/Q333373","display_name":"Abstract syntax","level":3,"score":0.41047194600105286},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.28686290979385376},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3510454.3528648","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3510454.3528648","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM/IEEE 44th International Conference on Software Engineering: Companion Proceedings","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.5699999928474426}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2945332763","https://openalex.org/W2950599942","https://openalex.org/W2963868406","https://openalex.org/W3104907093","https://openalex.org/W3125781052","https://openalex.org/W6600926619"],"related_works":["https://openalex.org/W2536864162","https://openalex.org/W3176913510","https://openalex.org/W3138016835","https://openalex.org/W4310447811","https://openalex.org/W3012064729","https://openalex.org/W2786676889","https://openalex.org/W2388254906","https://openalex.org/W2955426500","https://openalex.org/W4213130063","https://openalex.org/W3207813833"],"abstract_inverted_index":{"Recent":[0],"work":[1,119,177],"has":[2,55],"focused":[3],"on":[4,126,172],"using":[5],"machine":[6],"learning":[7],"to":[8,49,93,100,147,160],"automate":[9],"software":[10],"engineering":[11],"processes,":[12],"such":[13,141],"as":[14,61,142],"code":[15,17,21],"completion,":[16],"migration,":[18],"and":[19,71,87,156,178,192,201],"generating":[20],"from":[22,114,175],"natural":[23,94],"language":[24,70],"description.":[25],"One":[26],"of":[27,38,112,139,164,182],"the":[28,36,39,47,130,136,162,165,197],"challenges":[29],"faced":[30],"in":[31,104,158],"these":[32],"tasks":[33],"is":[34,42],"evaluating":[35],"quality":[37],"predictions,":[40],"which":[41,107],"usually":[43],"done":[44],"by":[45],"comparing":[46],"prediction":[48],"a":[50,179,203],"reference":[51],"solution.":[52],"BLEU":[53,99,199],"score":[54,200],"been":[56],"adopted":[57],"for":[58,67,207],"programming":[59,69,82,149],"languages":[60,83],"it":[62],"can":[63],"be":[64],"easily":[65],"computed":[66],"any":[68],"even":[72],"incomplete":[73],"source":[74,208],"code,":[75,145,209],"while":[76],"enabling":[77],"fast":[78],"automated":[79],"evaluation.":[80],"However,":[81],"are":[84],"more":[85],"verbose":[86],"have":[88],"strict":[89],"syntax":[90],"when":[91],"compared":[92],"languages.":[95],"This":[96,118],"feature":[97],"causes":[98],"find":[101],"common":[102],"n-grams":[103],"unrelated":[105,193],"programs,":[106],"makes":[108],"distinguishing":[109],"similar":[110,191],"pairs":[111,116],"programs":[113,194],"dissimilar":[115],"hard.":[117],"presents":[120],"CrystalBLEU,":[121],"an":[122],"evaluation":[123],"metric":[124,134],"based":[125],"BLEU,":[127,140],"that":[128,188],"mitigates":[129],"distinguishability":[131],"problem.":[132],"Our":[133,185],"maintains":[135],"desirable":[137],"properties":[138],"handling":[143],"partial":[144],"applicability":[146],"all":[148],"languages,":[150],"high":[151],"correlation":[152],"with":[153],"human":[154],"judgement,":[155],"efficiency,":[157],"addition":[159],"reducing":[161],"effects":[163],"trivially":[166],"matched":[167],"n-grams.":[168],"We":[169],"evaluate":[170],"CrystalBLEU":[171,189],"two":[173],"datasets":[174],"previous":[176],"new":[180],"dataset":[181],"human-written":[183],"code.":[184],"results":[186],"show":[187],"differentiates":[190],"better":[195],"than":[196],"original":[198],"also":[202],"variant":[204],"designed":[205],"specifically":[206],"CodeBLEU.":[210]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2023-01-05T00:00:00"}