{"id":"https://openalex.org/W7135079487","doi":"https://doi.org/10.48550/arxiv.2603.09998","title":"Automated evaluation of LLMs for effective machine translation of Mandarin Chinese to English","display_name":"Automated evaluation of LLMs for effective machine translation of Mandarin Chinese to English","publication_year":2026,"publication_date":"2026-02-15","ids":{"openalex":"https://openalex.org/W7135079487","doi":"https://doi.org/10.48550/arxiv.2603.09998"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.09998","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09998","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.09998","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128842443","display_name":"Yue Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068060736","display_name":"Rodney Beard","orcid":"https://orcid.org/0000-0001-5506-5424"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Beard, Rodney","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128917209","display_name":"John Hawkins","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hawkins, John","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128850405","display_name":"Rohitash Chandra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chandra, Rohitash","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2831000089645386,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2831000089645386,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.08829999715089798,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.08810000121593475,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mandarin-chinese","display_name":"Mandarin Chinese","score":0.7759000062942505},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.6854000091552734},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5706999897956848},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.46939998865127563},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.40779998898506165},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.3977000117301941},{"id":"https://openalex.org/keywords/rule-based-machine-translation","display_name":"Rule-based machine translation","score":0.39340001344680786}],"concepts":[{"id":"https://openalex.org/C138954614","wikidata":"https://www.wikidata.org/wiki/Q9192","display_name":"Mandarin Chinese","level":2,"score":0.7759000062942505},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.6854000091552734},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6639999747276306},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6406999826431274},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6308000087738037},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5706999897956848},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.46939998865127563},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.40779998898506165},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.3977000117301941},{"id":"https://openalex.org/C53893814","wikidata":"https://www.wikidata.org/wiki/Q7378909","display_name":"Rule-based machine translation","level":2,"score":0.39340001344680786},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.3711000084877014},{"id":"https://openalex.org/C46182478","wikidata":"https://www.wikidata.org/wiki/Q7363315","display_name":"Literal and figurative language","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.35929998755455017},{"id":"https://openalex.org/C2777761643","wikidata":"https://www.wikidata.org/wiki/Q1191837","display_name":"Literal translation","level":3,"score":0.35530000925064087},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3310000002384186},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C135784402","wikidata":"https://www.wikidata.org/wiki/Q6958279","display_name":"Evaluation of machine translation","level":5,"score":0.30959999561309814},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.28630000352859497},{"id":"https://openalex.org/C144559511","wikidata":"https://www.wikidata.org/wiki/Q2986279","display_name":"Principal (computer security)","level":2,"score":0.2540000081062317}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.09998","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09998","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.09998","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.09998","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.8606711626052856,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Although":[0,165],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"have":[5],"exceptional":[6],"performance":[7,159,179],"in":[8,25,92,150,157,173,180,191],"machine":[9,61],"translation,":[10,153],"only":[11],"a":[12,42],"limited":[13],"systematic":[14],"assessment":[15],"of":[16,45,51,95,128],"translation":[17,52,75,192],"quality":[18,127],"has":[19],"been":[20],"done.":[21],"The":[22],"challenge":[23],"lies":[24],"automated":[26,60],"frameworks,":[27],"as":[28,109,111],"human-expert-based":[29],"evaluations":[30],"can":[31],"be":[32],"time-consuming,":[33],"given":[34],"the":[35,39,115,126,146,188,208],"fast-evolving":[36],"LLMs":[37,132,147],"and":[38,66,79,84,89,106,133,167,184,199],"need":[40],"for":[41,206],"diverse":[43],"set":[44],"texts":[46,91,102],"to":[47,69,73,124,162],"ensure":[48],"fair":[49],"assessments":[50],"quality.":[53],"In":[54],"this":[55],"paper,":[56],"we":[57,119],"utilise":[58,120],"an":[59,138,203],"learning":[62],"framework":[63],"featuring":[64],"semantic":[65,171],"sentiment":[67],"analysis":[68],"assess":[70],"Mandarin":[71],"Chinese":[72,97],"English":[74],"using":[76],"Google":[77],"Translate":[78],"LLMs,":[80],"including":[81],"GPT-4,":[82],"GPT-4o,":[83],"DeepSeek.":[85],"We":[86],"compare":[87,125],"original":[88],"translated":[90],"various":[93],"classes":[94],"high-profile":[96],"texts,":[98],"which":[99],"include":[100],"novel":[101,121],"that":[103,145],"span":[104],"modern":[105],"classical":[107,197],"literature,":[108],"well":[110,149],"news":[112,151],"articles.":[113],"As":[114],"main":[116],"evaluation":[117],"measures,":[118],"similarity":[122],"metrics":[123],"translations":[129],"produced":[130],"by":[131,137],"further":[134],"evaluate":[135],"them":[136],"expert":[139],"human":[140],"translator.":[141],"Our":[142],"results":[143],"indicate":[144],"perform":[148],"media":[152],"but":[154],"show":[155],"divergence":[156],"their":[158],"when":[160],"applied":[161],"literary":[163],"texts.":[164],"GPT-4o":[166],"DeepSeek":[168,176],"demonstrated":[169,177],"better":[170,178],"conservation":[172],"complex":[174],"situations,":[175],"preserving":[181],"cultural":[182,195],"subtleties":[183],"grammatical":[185],"rendering.":[186],"Nevertheless,":[187],"subtle":[189],"challenges":[190],"remain:":[193],"maintaining":[194],"details,":[196],"references":[198],"figurative":[200],"expressions":[201],"remain":[202],"open":[204],"problem":[205],"all":[207],"models.":[209]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-13T00:00:00"}