{"id":"https://openalex.org/W3195729941","doi":"https://doi.org/10.48550/arxiv.2108.06665","title":"Accurate, yet inconsistent? Consistency Analysis on Language Understanding Models","display_name":"Accurate, yet inconsistent? Consistency Analysis on Language Understanding Models","publication_year":2021,"publication_date":"2021-08-15","ids":{"openalex":"https://openalex.org/W3195729941","doi":"https://doi.org/10.48550/arxiv.2108.06665","mag":"3195729941"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2108.06665","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.06665","pdf_url":"https://arxiv.org/pdf/2108.06665","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2108.06665","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055458201","display_name":"Myeongjun Jang","orcid":"https://orcid.org/0000-0002-9352-4799","observed_orcids":["https://orcid.org/0000-0002-9352-4799"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jang, Myeongjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011790128","display_name":"Deuk Sin Kwon","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kwon, Deuk Sin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5091549352","display_name":"Thomas Lukasiewicz","orcid":"https://orcid.org/0000-0002-7644-1668","observed_orcids":["https://orcid.org/0000-0002-7644-1668","https://orcid.org/0009-0003-9914-0416"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lukasiewicz, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9463000297546387,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9463000297546387,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.043299999088048935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.001500000013038516,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/consistency-analysis","display_name":"consistency analysis","score":0.7749999761581421},{"id":"https://openalex.org/keywords/pre-trained-language-models","display_name":"pre-trained language models","score":0.9459999799728394},{"id":"https://openalex.org/keywords/semantic-similarity","display_name":"semantic similarity","score":0.5529999732971191},{"id":"https://openalex.org/keywords/paraphrase-detection","display_name":"paraphrase detection","score":0.8659999966621399},{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"multi-task learning","score":0.9380000233650208},{"id":"https://openalex.org/keywords/model-evaluation","display_name":"model evaluation","score":0.47699999809265137}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6874904632568359},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4233981966972351},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3687022030353546},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3661932349205017},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3613200783729553},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21963226795196533},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.10802733898162842}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2108.06665","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.06665","pdf_url":"https://arxiv.org/pdf/2108.06665","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2108.06665","doi":"https://doi.org/10.48550/arxiv.2108.06665","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2108.06665","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2108.06665","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.06665","pdf_url":"https://arxiv.org/pdf/2108.06665","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.6399999856948853}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W131533222","https://openalex.org/W2607892599","https://openalex.org/W2621144345","https://openalex.org/W2768282280","https://openalex.org/W2788292930","https://openalex.org/W2951286828","https://openalex.org/W2952984539","https://openalex.org/W2953039212","https://openalex.org/W2963310665","https://openalex.org/W2963341956","https://openalex.org/W2963394326","https://openalex.org/W2963748441","https://openalex.org/W2963854351","https://openalex.org/W2965647350","https://openalex.org/W2968297680","https://openalex.org/W2972381305","https://openalex.org/W2981456979","https://openalex.org/W2996851481","https://openalex.org/W3011574394","https://openalex.org/W3014153848","https://openalex.org/W3023533951","https://openalex.org/W3025064182","https://openalex.org/W3030163527","https://openalex.org/W3034649382","https://openalex.org/W3034723486","https://openalex.org/W3035068109","https://openalex.org/W3098267758","https://openalex.org/W3099843385","https://openalex.org/W3101449015","https://openalex.org/W3102927624","https://openalex.org/W3102999298","https://openalex.org/W3105604018","https://openalex.org/W3110879614","https://openalex.org/W3119077874","https://openalex.org/W3119438769","https://openalex.org/W3119555019","https://openalex.org/W3127389184","https://openalex.org/W3130319171","https://openalex.org/W3152736378","https://openalex.org/W3152956381","https://openalex.org/W3159613532","https://openalex.org/W3162296828","https://openalex.org/W3169948074","https://openalex.org/W3202712981","https://openalex.org/W4393693013"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W4391913857","https://openalex.org/W2350741829"],"abstract_inverted_index":{"Consistency,":[0],"which":[1],"refers":[2],"to":[3,66,82,104],"the":[4,8,45,68,108],"capability":[5],"of":[6,102],"generating":[7],"same":[9],"predictions":[10,85],"for":[11,20,87],"semantically":[12,88],"similar":[13],"contexts,":[14],"is":[15,101],"a":[16,21,55],"highly":[17],"desirable":[18],"property":[19],"sound":[22],"language":[23,29,62],"understanding":[24,63],"model.":[25],"Although":[26],"recent":[27],"pretrained":[28],"models":[30,46,64],"(PLMs)":[31],"deliver":[32],"outstanding":[33],"performance":[34],"in":[35],"various":[36],"downstream":[37],"tasks,":[38],"they":[39],"should":[40],"exhibit":[41],"consistent":[42],"behaviour":[43],"provided":[44],"truly":[47],"understand":[48],"language.":[49],"In":[50],"this":[51],"paper,":[52],"we":[53,75],"propose":[54],"simple":[56],"framework":[57],"named":[58],"consistency":[59,71,109],"analysis":[60],"on":[61,112],"(CALUM)}":[65],"evaluate":[67],"model's":[69],"lower-bound":[70],"ability.":[72],"Through":[73],"experiments,":[74],"confirmed":[76],"that":[77,94],"current":[78],"PLMs":[79],"are":[80],"prone":[81],"generate":[83],"inconsistent":[84],"even":[86],"identical":[89],"inputs.":[90],"We":[91],"also":[92],"observed":[93],"multi-task":[95],"training":[96],"with":[97],"paraphrase":[98],"identification":[99],"tasks":[100],"benefit":[103],"improve":[105],"consistency,":[106],"increasing":[107],"by":[110],"13%":[111],"average.":[113]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":3}],"updated_date":"2026-10-12T09:40:04.860316","created_date":"2021-08-30T00:00:00"}