{"id":"https://openalex.org/W4409426851","doi":"https://doi.org/10.1109/ojcs.2025.3560333","title":"A Detailed Comparative Analysis of Automatic Neural Metrics for Machine Translation: BLEURT &amp; BERTScore","display_name":"A Detailed Comparative Analysis of Automatic Neural Metrics for Machine Translation: BLEURT &amp; BERTScore","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4409426851","doi":"https://doi.org/10.1109/ojcs.2025.3560333"},"language":"en","primary_location":{"id":"doi:10.1109/ojcs.2025.3560333","is_oa":true,"landing_page_url":"https://doi.org/10.1109/ojcs.2025.3560333","pdf_url":null,"source":{"id":"https://openalex.org/S4210176459","display_name":"IEEE Open Journal of the Computer Society","issn_l":"2644-1268","issn":["2644-1268"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Open Journal of the Computer Society","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/ojcs.2025.3560333","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Aniruddha Mukherjee","orcid":"https://orcid.org/0009-0002-1273-8558"},"institutions":[{"id":"https://openalex.org/I67357951","display_name":"KIIT University","ror":"https://ror.org/00k8zt527","country_code":"IN","type":"education","lineage":["https://openalex.org/I67357951"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Aniruddha Mukherjee","raw_affiliation_strings":["School of Computer Engineering, Kalinga Institute of Industrial Technology (KIIT) Deemed to be University, Bhubaneswar, Odisha, India","School of Computer Engineering, KIIT, Bhubaneshwar, India"],"raw_orcid":"https://orcid.org/0009-0002-1273-8558","affiliations":[{"raw_affiliation_string":"School of Computer Engineering, Kalinga Institute of Industrial Technology (KIIT) Deemed to be University, Bhubaneswar, Odisha, India","institution_ids":["https://openalex.org/I67357951"]},{"raw_affiliation_string":"School of Computer Engineering, KIIT, Bhubaneshwar, India","institution_ids":["https://openalex.org/I67357951"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Vikas Hassija","orcid":"https://orcid.org/0009-0003-9023-1661"},"institutions":[{"id":"https://openalex.org/I67357951","display_name":"KIIT University","ror":"https://ror.org/00k8zt527","country_code":"IN","type":"education","lineage":["https://openalex.org/I67357951"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Vikas Hassija","raw_affiliation_strings":["School of Computer Engineering, Kalinga Institute of Industrial Technology (KIIT) Deemed to be University, Bhubaneswar, Odisha, India","School of Computer Engineering, KIIT, Bhubaneshwar, India"],"raw_orcid":"https://orcid.org/0009-0003-9023-1661","affiliations":[{"raw_affiliation_string":"School of Computer Engineering, Kalinga Institute of Industrial Technology (KIIT) Deemed to be University, Bhubaneswar, Odisha, India","institution_ids":["https://openalex.org/I67357951"]},{"raw_affiliation_string":"School of Computer Engineering, KIIT, Bhubaneshwar, India","institution_ids":["https://openalex.org/I67357951"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005020243","display_name":"Vinay Chamola","orcid":"https://orcid.org/0000-0002-6730-3060"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Vinay Chamola","raw_affiliation_strings":["Department of Electrical and Electronics Engineering, Birla Institute of Technology and Science, Pilani, Pilani Campus, Vidya Vihar, Pilani, Rajasthan, India","Department of Electrical and Electronics Engineering, BITS-Pilani, Pilani Campus, Rajasthan, India"],"raw_orcid":"https://orcid.org/0000-0002-6730-3060","affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronics Engineering, Birla Institute of Technology and Science, Pilani, Pilani Campus, Vidya Vihar, Pilani, Rajasthan, India","institution_ids":["https://openalex.org/I74796645"]},{"raw_affiliation_string":"Department of Electrical and Electronics Engineering, BITS-Pilani, Pilani Campus, Rajasthan, India","institution_ids":["https://openalex.org/I74796645"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008754589","display_name":"Karunesh Kumar Gupta","orcid":"https://orcid.org/0000-0002-0003-4601"},"institutions":[{"id":"https://openalex.org/I74796645","display_name":"Birla Institute of Technology and Science, Pilani","ror":"https://ror.org/001p3jz28","country_code":"IN","type":"education","lineage":["https://openalex.org/I74796645"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Karunesh Kumar Gupta","raw_affiliation_strings":["Department of Electrical and Electronics Engineering, Birla Institute of Technology and Science, Pilani, Pilani Campus, Vidya Vihar, Pilani, Rajasthan, India","Department of Electrical and Electronics Engineering, BITS-Pilani, Pilani Campus, Rajasthan, India"],"raw_orcid":"https://orcid.org/0000-0002-0003-4601","affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronics Engineering, Birla Institute of Technology and Science, Pilani, Pilani Campus, Vidya Vihar, Pilani, Rajasthan, India","institution_ids":["https://openalex.org/I74796645"]},{"raw_affiliation_string":"Department of Electrical and Electronics Engineering, BITS-Pilani, Pilani Campus, Rajasthan, India","institution_ids":["https://openalex.org/I74796645"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I67357951"],"apc_list":{"value":1750,"currency":"USD","value_usd":1750},"apc_paid":{"value":1750,"currency":"USD","value_usd":1750},"fwci":10.9584,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.97938686,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"6","issue":null,"first_page":"658","last_page":"668"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.6401000022888184,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.6401000022888184,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5712789297103882},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.5584664344787598},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4889622926712036},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.4810336232185364},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.32308655977249146},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.23975586891174316},{"id":"https://openalex.org/keywords/biochemistry","display_name":"Biochemistry","score":0.07311597466468811}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5712789297103882},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.5584664344787598},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4889622926712036},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.4810336232185364},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32308655977249146},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.23975586891174316},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.07311597466468811},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/ojcs.2025.3560333","is_oa":true,"landing_page_url":"https://doi.org/10.1109/ojcs.2025.3560333","pdf_url":null,"source":{"id":"https://openalex.org/S4210176459","display_name":"IEEE Open Journal of the Computer Society","issn_l":"2644-1268","issn":["2644-1268"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Open Journal of the Computer Society","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:917ee0ef027a41f1b6bee08d9e6efde9","is_oa":true,"landing_page_url":"https://doaj.org/article/917ee0ef027a41f1b6bee08d9e6efde9","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Open Journal of the Computer Society, Vol 6, Pp 658-668 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/ojcs.2025.3560333","is_oa":true,"landing_page_url":"https://doi.org/10.1109/ojcs.2025.3560333","pdf_url":null,"source":{"id":"https://openalex.org/S4210176459","display_name":"IEEE Open Journal of the Computer Society","issn_l":"2644-1268","issn":["2644-1268"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Open Journal of the Computer Society","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W1608789752","https://openalex.org/W1663984431","https://openalex.org/W2098297786","https://openalex.org/W2101105183","https://openalex.org/W2250591774","https://openalex.org/W2576882021","https://openalex.org/W2806532810","https://openalex.org/W2884001105","https://openalex.org/W2896457183","https://openalex.org/W2896780650","https://openalex.org/W2906152891","https://openalex.org/W2946359678","https://openalex.org/W2953166927","https://openalex.org/W2955770773","https://openalex.org/W2963903950","https://openalex.org/W2970120757","https://openalex.org/W2982756474","https://openalex.org/W3004346089","https://openalex.org/W3034503989","https://openalex.org/W3035252911","https://openalex.org/W3037636427","https://openalex.org/W3102783139","https://openalex.org/W3103769626","https://openalex.org/W3105214104","https://openalex.org/W3105661746","https://openalex.org/W3118485687","https://openalex.org/W3133702157","https://openalex.org/W3155682407","https://openalex.org/W3174169056","https://openalex.org/W3187417346","https://openalex.org/W4285306212","https://openalex.org/W4385245566","https://openalex.org/W6629155019","https://openalex.org/W6631167871","https://openalex.org/W6652311901","https://openalex.org/W6678262379","https://openalex.org/W6679434410","https://openalex.org/W6679436768","https://openalex.org/W6682631176","https://openalex.org/W6761205521","https://openalex.org/W6784395031","https://openalex.org/W6787040549","https://openalex.org/W6791385579","https://openalex.org/W6846253096","https://openalex.org/W6854162061","https://openalex.org/W7000626104"],"related_works":["https://openalex.org/W2775554247","https://openalex.org/W2883671469","https://openalex.org/W2728761353","https://openalex.org/W2110168585","https://openalex.org/W3107474891","https://openalex.org/W2250213760","https://openalex.org/W4386247111","https://openalex.org/W4327642362","https://openalex.org/W2587014613","https://openalex.org/W3204019825"],"abstract_inverted_index":{"<bold":[0],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[1,10,40,47,51,72,121,141,149,168,186,202],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><small>Bleurt</small></b>":[2],"a":[3,12,25],"recently":[4],"introduced":[5],"metric":[6,163],"that":[7,70,174,184],"employs":[8],"<sc":[9,39,46,50,71,120,140,148,167,185,201],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Bert</small>,":[11,73],"potent":[13],"pre-trained":[14],"language":[15,80],"model":[16],"to":[17,24,57],"assess":[18],"how":[19,119,143,154],"well":[20],"candidate":[21],"translations":[22],"compare":[23,157],"reference":[26],"translation":[27,33,125,217],"in":[28,78,90,215],"the":[29,101,135,159],"context":[30],"of":[31,106,139],"machine":[32,165,216],"outputs.":[34],"While":[35],"traditional":[36],"metrics":[37],"like":[38],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Bleu</small>":[41],"rely":[42],"on":[43],"lexical":[44,110],"similarities,":[45],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Bleurt</small>":[48,122,187],"leverages":[49],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Bert</small>'s":[52,150],"semantic":[53,94],"and":[54,93,114,137,153,179],"syntactic":[55,92],"capabilities":[56],"provide":[58,209],"more":[59],"robust":[60],"evaluation":[61],"through":[62],"complex":[63],"text":[64],"representations.":[65],"However,":[66],"studies":[67],"have":[68],"shown":[69],"despite":[74],"its":[75],"impressive":[76],"performance":[77],"natural":[79],"processing":[81],"tasks":[82],"can":[83],"sometimes":[84],"deviate":[85],"from":[86],"human":[87],"judgment,":[88],"particularly":[89],"specific":[91],"scenarios.":[95],"Through":[96],"systematic":[97,207],"experimental":[98],"analysis":[99],"at":[100,189],"word":[102],"level,":[103],"including":[104],"categorization":[105],"errors":[107],"such":[108],"as":[109],"mismatches,":[111],"untranslated":[112],"terms,":[113],"structural":[115],"inconsistencies,":[116],"we":[117,182],"investigate":[118],"handles":[123],"various":[124],"challenges.":[126],"Our":[127,206],"study":[128],"addresses":[129],"three":[130],"central":[131],"questions:":[132],"What":[133],"are":[134],"strengths":[136],"weaknesses":[138],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Bleurt</small>,":[142],"do":[144],"they":[145],"align":[146],"with":[147,158,195],"known":[151],"limitations,":[152],"does":[155],"it":[156],"similar":[160],"automatic":[161],"neural":[162],"for":[164,211],"translation,":[166],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">BERTScore</small>?":[169],"Using":[170],"manually":[171],"annotated":[172],"datasets":[173],"emphasize":[175],"different":[176],"error":[177],"types":[178],"linguistic":[180],"phenomena,":[181],"find":[183],"excels":[188],"identifying":[190],"nuanced":[191],"differences":[192],"between":[193],"sentences":[194],"high":[196],"overlap,":[197],"an":[198],"area":[199],"where":[200],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">BERTScore</small>":[203],"shows":[204],"limitations.":[205],"experiments,":[208],"insights":[210],"their":[212],"effective":[213],"application":[214],"evaluation.":[218]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4}],"updated_date":"2026-06-03T09:05:47.796612","created_date":"2025-10-10T00:00:00"}