{"id":"https://openalex.org/W3100869665","doi":"https://doi.org/10.1145/3368089.3409756","title":"Machine translation testing via pathological invariance","display_name":"Machine translation testing via pathological invariance","publication_year":2020,"publication_date":"2020-11-08","ids":{"openalex":"https://openalex.org/W3100869665","doi":"https://doi.org/10.1145/3368089.3409756","mag":"3100869665"},"language":"en","primary_location":{"id":"doi:10.1145/3368089.3409756","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3368089.3409756","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077853405","display_name":"Shashij Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Shashij Gupta","raw_affiliation_strings":["IIT Bombay, India"],"affiliations":[{"raw_affiliation_string":"IIT Bombay, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101941950","display_name":"Pinjia He","orcid":"https://orcid.org/0000-0003-3377-8129"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Pinjia He","raw_affiliation_strings":["ETH Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Zurich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029033876","display_name":"Clara Meister","orcid":"https://orcid.org/0000-0002-3775-4426"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Clara Meister","raw_affiliation_strings":["ETH Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Zurich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077610917","display_name":"Zhendong Su","orcid":"https://orcid.org/0000-0002-2970-1391"},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Zhendong Su","raw_affiliation_strings":["ETH Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"ETH Zurich, Switzerland","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5077853405"],"corresponding_institution_ids":["https://openalex.org/I162827531"],"apc_list":null,"apc_paid":null,"fwci":3.2917,"has_fulltext":false,"cited_by_count":43,"citation_normalized_percentile":{"value":0.93568721,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"863","last_page":"875"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.8547337055206299},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8409295082092285},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.752602756023407},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6888753175735474},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6751269102096558},{"id":"https://openalex.org/keywords/phrase","display_name":"Phrase","score":0.6431583166122437},{"id":"https://openalex.org/keywords/machine-translation-software-usability","display_name":"Machine translation software usability","score":0.5675211548805237},{"id":"https://openalex.org/keywords/example-based-machine-translation","display_name":"Example-based machine translation","score":0.49522319436073303},{"id":"https://openalex.org/keywords/language-translation","display_name":"Language translation","score":0.4806308150291443},{"id":"https://openalex.org/keywords/rule-based-machine-translation","display_name":"Rule-based machine translation","score":0.474685400724411},{"id":"https://openalex.org/keywords/intuition","display_name":"Intuition","score":0.4429266154766083}],"concepts":[{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.8547337055206299},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8409295082092285},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.752602756023407},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6888753175735474},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6751269102096558},{"id":"https://openalex.org/C2776224158","wikidata":"https://www.wikidata.org/wiki/Q187931","display_name":"Phrase","level":2,"score":0.6431583166122437},{"id":"https://openalex.org/C148526163","wikidata":"https://www.wikidata.org/wiki/Q6723733","display_name":"Machine translation software usability","level":4,"score":0.5675211548805237},{"id":"https://openalex.org/C24687705","wikidata":"https://www.wikidata.org/wiki/Q3753284","display_name":"Example-based machine translation","level":3,"score":0.49522319436073303},{"id":"https://openalex.org/C2986862884","wikidata":"https://www.wikidata.org/wiki/Q7553","display_name":"Language translation","level":3,"score":0.4806308150291443},{"id":"https://openalex.org/C53893814","wikidata":"https://www.wikidata.org/wiki/Q7378909","display_name":"Rule-based machine translation","level":2,"score":0.474685400724411},{"id":"https://openalex.org/C132010649","wikidata":"https://www.wikidata.org/wiki/Q189222","display_name":"Intuition","level":2,"score":0.4429266154766083},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3368089.3409756","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3368089.3409756","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.6899999976158142}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W1982483659","https://openalex.org/W2019108269","https://openalex.org/W2041650849","https://openalex.org/W2101105183","https://openalex.org/W2123442489","https://openalex.org/W2250539671","https://openalex.org/W2293505944","https://openalex.org/W2324595780","https://openalex.org/W2483327705","https://openalex.org/W2560674852","https://openalex.org/W2607219512","https://openalex.org/W2616028256","https://openalex.org/W2741609678","https://openalex.org/W2782311202","https://openalex.org/W2804337238","https://openalex.org/W2888307014","https://openalex.org/W2898868990","https://openalex.org/W2905526464","https://openalex.org/W2922293812","https://openalex.org/W2947133760","https://openalex.org/W2950577311","https://openalex.org/W2954629067","https://openalex.org/W2954903132","https://openalex.org/W2957066083","https://openalex.org/W2957905354","https://openalex.org/W2962718684","https://openalex.org/W2962739339","https://openalex.org/W2962818872","https://openalex.org/W2963126845","https://openalex.org/W2963216553","https://openalex.org/W2963327228","https://openalex.org/W2963615251","https://openalex.org/W2963856968","https://openalex.org/W2963859254","https://openalex.org/W2963969878","https://openalex.org/W2964082701","https://openalex.org/W2964164993","https://openalex.org/W2968940383","https://openalex.org/W2969499518","https://openalex.org/W2997653900","https://openalex.org/W3007157104","https://openalex.org/W3090988182","https://openalex.org/W3091407209","https://openalex.org/W3099444373","https://openalex.org/W3102720581","https://openalex.org/W3105599650","https://openalex.org/W4229781645","https://openalex.org/W4246553962"],"related_works":["https://openalex.org/W3214198970","https://openalex.org/W2937574835","https://openalex.org/W2086774483","https://openalex.org/W2027317339","https://openalex.org/W3097573162","https://openalex.org/W4362495644","https://openalex.org/W1987510781","https://openalex.org/W2892009878","https://openalex.org/W4312364893","https://openalex.org/W4306816465"],"abstract_inverted_index":{"Machine":[0],"translation":[1,25,60],"software":[2,26],"has":[3,27],"become":[4],"heavily":[5],"integrated":[6],"into":[7],"our":[8],"daily":[9],"lives":[10],"due":[11,49],"to":[12,30,38,50,163],"the":[13,17,51,54,90,121,136,147,151,154,214],"recent":[14],"improvement":[15],"in":[16,120,196,203],"performance":[18],"of":[19,53,101,208],"deep":[20],"neural":[21,56],"networks.":[22],"However,":[23],"machine":[24,59],"been":[28],"shown":[29],"regularly":[31],"return":[32],"erroneous":[33,194,201],"translations,":[34],"which":[35,209],"can":[36,190],"lead":[37],"harmful":[39],"consequences":[40],"such":[41],"as":[42],"economic":[43],"loss":[44],"and":[45,156,167,182,199],"political":[46],"conflicts.":[47],"Additionally,":[48],"complexity":[52],"underlying":[55],"models,":[57],"testing":[58],"systems":[61],"presents":[62],"new":[63],"challenges.":[64],"To":[65],"address":[66],"this":[67,94],"problem,":[68],"we":[69,97],"introduce":[70],"a":[71,124],"novel":[72],"methodology":[73],"called":[74],"PatInv.":[75],"The":[76,185],"main":[77],"intuition":[78],"behind":[79],"PatInv":[80,102,162,189],"is":[81],"that":[82,103,188],"sentences":[83,114],"with":[84],"different":[85,113],"meanings":[86],"should":[87],"not":[88],"have":[89,160],"same":[91,152],"translation.":[92],"Under":[93],"general":[95],"idea,":[96],"provide":[98],"two":[99],"realizations":[100],"given":[104],"an":[105],"arbitrary":[106],"sentence,":[107],"generate":[108],"syntactically":[109],"similar":[110],"but":[111],"semantically":[112],"by:":[115],"(1)":[116],"replacing":[117],"one":[118,131],"word":[119,132],"sentence":[122,137],"using":[123,171],"masked":[125],"language":[126,176],"model":[127],"or":[128,133],"(2)":[129],"removing":[130],"phrase":[134],"from":[135],"based":[138],"on":[139],"its":[140],"constituency":[141],"structure.":[142],"We":[143,159],"then":[144],"test":[145,164],"whether":[146],"returned":[148],"translations":[149,195,202],"are":[150,178],"for":[153],"original":[155],"modified":[157],"sentences.":[158,174],"applied":[161],"Google":[165,197],"Translate":[166,198],"Bing":[168,204],"Microsoft":[169,205],"Translator":[170],"200":[172],"English":[173],"Two":[175],"settings":[177],"considered:":[179],"English-Hindi":[180],"(En-Hi)":[181],"English-Chinese":[183],"(En-Zh).":[184],"results":[186],"show":[187],"accurately":[191],"find":[192],"308":[193],"223":[200],"Translator,":[206],"most":[207],"cannot":[210],"be":[211],"found":[212],"by":[213],"state-of-the-art":[215],"approaches.":[216]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":11},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":1}],"updated_date":"2026-03-17T09:09:15.849793","created_date":"2025-10-10T00:00:00"}