{"id":"https://openalex.org/W2927167868","doi":"https://doi.org/10.18653/v1/s19-2220","title":"ThisIsCompetition at SemEval-2019 Task 9: BERT is unstable for out-of-domain samples","display_name":"ThisIsCompetition at SemEval-2019 Task 9: BERT is unstable for out-of-domain samples","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2927167868","doi":"https://doi.org/10.18653/v1/s19-2220","mag":"2927167868"},"language":"en","primary_location":{"id":"doi:10.18653/v1/s19-2220","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/s19-2220","pdf_url":"https://www.aclweb.org/anthology/S19-2220.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th International Workshop on Semantic Evaluation","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/S19-2220.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009313002","display_name":"Cheoneum Park","orcid":"https://orcid.org/0000-0001-5386-0483"},"institutions":[{"id":"https://openalex.org/I165507594","display_name":"Kangwon National University","ror":"https://ror.org/01mh5ph17","country_code":"KR","type":"education","lineage":["https://openalex.org/I165507594"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Cheoneum Park","raw_affiliation_strings":["Kangwon National University, South Korea","Kangwon National University, Chuncheon, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kangwon National University, South Korea","institution_ids":["https://openalex.org/I165507594"]},{"raw_affiliation_string":"Kangwon National University, Chuncheon, South Korea","institution_ids":["https://openalex.org/I165507594"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057884782","display_name":"Juae Kim","orcid":"https://orcid.org/0000-0001-7826-5226"},"institutions":[{"id":"https://openalex.org/I148751991","display_name":"Sogang University","ror":"https://ror.org/056tn4839","country_code":"KR","type":"education","lineage":["https://openalex.org/I148751991"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Juae Kim","raw_affiliation_strings":["Sogang University, South Korea","Sogang University, Seoul, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sogang University, South Korea","institution_ids":["https://openalex.org/I148751991"]},{"raw_affiliation_string":"Sogang University, Seoul, South Korea","institution_ids":["https://openalex.org/I148751991"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045131332","display_name":"Hyeon-gu Lee","orcid":"https://orcid.org/0000-0001-5702-7931"},"institutions":[{"id":"https://openalex.org/I165507594","display_name":"Kangwon National University","ror":"https://ror.org/01mh5ph17","country_code":"KR","type":"education","lineage":["https://openalex.org/I165507594"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Hyeon-gu Lee","raw_affiliation_strings":["Kangwon National University, South Korea","Kangwon National University, Chuncheon, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kangwon National University, South Korea","institution_ids":["https://openalex.org/I165507594"]},{"raw_affiliation_string":"Kangwon National University, Chuncheon, South Korea","institution_ids":["https://openalex.org/I165507594"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009993083","display_name":"Reinald Kim Amplayo","orcid":null},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Reinald Kim Amplayo","raw_affiliation_strings":["University of Edinburgh, UK","University of Edinburgh, Edinburgh, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh, UK","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"University of Edinburgh, Edinburgh, United Kingdom","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022865376","display_name":"Harksoo Kim","orcid":"https://orcid.org/0000-0002-8286-7198"},"institutions":[{"id":"https://openalex.org/I165507594","display_name":"Kangwon National University","ror":"https://ror.org/01mh5ph17","country_code":"KR","type":"education","lineage":["https://openalex.org/I165507594"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Harksoo Kim","raw_affiliation_strings":["Kangwon National University, South Korea","Kangwon National University, Chuncheon, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kangwon National University, South Korea","institution_ids":["https://openalex.org/I165507594"]},{"raw_affiliation_string":"Kangwon National University, Chuncheon, South Korea","institution_ids":["https://openalex.org/I165507594"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101975931","display_name":"Jungyun Seo","orcid":"https://orcid.org/0000-0003-3670-7334"},"institutions":[{"id":"https://openalex.org/I148751991","display_name":"Sogang University","ror":"https://ror.org/056tn4839","country_code":"KR","type":"education","lineage":["https://openalex.org/I148751991"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jungyun Seo","raw_affiliation_strings":["Sogang University, South Korea","Sogang University, Seoul, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sogang University, South Korea","institution_ids":["https://openalex.org/I148751991"]},{"raw_affiliation_string":"Sogang University, Seoul, South Korea","institution_ids":["https://openalex.org/I148751991"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073790847","display_name":"Changki Lee","orcid":"https://orcid.org/0000-0001-8373-1500"},"institutions":[{"id":"https://openalex.org/I165507594","display_name":"Kangwon National University","ror":"https://ror.org/01mh5ph17","country_code":"KR","type":"education","lineage":["https://openalex.org/I165507594"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Changki Lee","raw_affiliation_strings":["Kangwon National University, South Korea","Kangwon National University, Chuncheon, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Kangwon National University, South Korea","institution_ids":["https://openalex.org/I165507594"]},{"raw_affiliation_string":"Kangwon National University, Chuncheon, South Korea","institution_ids":["https://openalex.org/I165507594"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1446,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.55569509,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"1254","last_page":"1261"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/semeval","display_name":"SemEval","score":0.845653772354126},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8101377487182617},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6959213614463806},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.6890657544136047},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6152930855751038},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5731835961341858},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.5665979981422424},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5539254546165466},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.522556483745575},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4556539058685303},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3583078980445862},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3408064842224121},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09806212782859802}],"concepts":[{"id":"https://openalex.org/C44572571","wikidata":"https://www.wikidata.org/wiki/Q7448970","display_name":"SemEval","level":3,"score":0.845653772354126},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8101377487182617},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6959213614463806},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.6890657544136047},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6152930855751038},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5731835961341858},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.5665979981422424},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5539254546165466},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.522556483745575},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4556539058685303},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3583078980445862},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3408064842224121},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09806212782859802},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/s19-2220","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/s19-2220","pdf_url":"https://www.aclweb.org/anthology/S19-2220.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th International Workshop on Semantic Evaluation","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1904.03339","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1904.03339","pdf_url":"https://arxiv.org/pdf/1904.03339","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2927167868","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1904.03339","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1904.03339","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1904.03339","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.18653/v1/s19-2220","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/s19-2220","pdf_url":"https://www.aclweb.org/anthology/S19-2220.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 13th International Workshop on Semantic Evaluation","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320328359","display_name":"Ministry of Science and ICT, South Korea","ror":"https://ror.org/01wpjm123"},{"id":"https://openalex.org/F4320335489","display_name":"Institute for Information and Communications Technology Promotion","ror":"https://ror.org/01g0hqq23"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2927167868.pdf","grobid_xml":"https://content.openalex.org/works/W2927167868.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W6908809","https://openalex.org/W22861983","https://openalex.org/W135195246","https://openalex.org/W163787259","https://openalex.org/W1603678606","https://openalex.org/W1731081199","https://openalex.org/W1832693441","https://openalex.org/W2095705004","https://openalex.org/W2097726431","https://openalex.org/W2108598243","https://openalex.org/W2108646579","https://openalex.org/W2123442489","https://openalex.org/W2125770490","https://openalex.org/W2128507180","https://openalex.org/W2153353890","https://openalex.org/W2167660864","https://openalex.org/W2250539671","https://openalex.org/W2251733102","https://openalex.org/W2399573548","https://openalex.org/W2513474745","https://openalex.org/W2567698949","https://openalex.org/W2594718649","https://openalex.org/W2911327180","https://openalex.org/W2912624765","https://openalex.org/W2912817604","https://openalex.org/W2954283529","https://openalex.org/W2962739339","https://openalex.org/W2963174729","https://openalex.org/W2963326042","https://openalex.org/W2963341956","https://openalex.org/W2963756346"],"related_works":["https://openalex.org/W2962829230","https://openalex.org/W2989276524","https://openalex.org/W3015356564","https://openalex.org/W2988736778","https://openalex.org/W3185606300","https://openalex.org/W2463861136","https://openalex.org/W3033187248","https://openalex.org/W3160799772","https://openalex.org/W2913518575","https://openalex.org/W3191747585","https://openalex.org/W2949541304","https://openalex.org/W2268463939","https://openalex.org/W2973095006","https://openalex.org/W2969810498","https://openalex.org/W2127293685","https://openalex.org/W2739458769","https://openalex.org/W2759035416","https://openalex.org/W2060786818","https://openalex.org/W3026842484","https://openalex.org/W2948335087"],"abstract_inverted_index":{"This":[0],"paper":[1],"describes":[2],"our":[3],"system,":[4],"Joint":[5],"Encoders":[6],"for":[7,12,73,85,98],"Stable":[8],"Suggestion":[9,18],"Inference":[10],"(JESSI),":[11],"the":[13,91],"SemEval":[14],"2019":[15],"Task":[16],"9:":[17],"Mining":[19],"from":[20,41,57],"Online":[21],"Reviews":[22],"and":[23,45,49,114,138],"Forums.":[24],"JESSI":[25],"is":[26,96,103],"a":[27,58,66,111],"combination":[28],"of":[29,54,90,122,142],"two":[30],"sentence":[31],"encoders:":[32],"(a)":[33],"one":[34,51],"using":[35,116,151],"multiple":[36],"pre-trained":[37,59],"word":[38,55],"embeddings":[39],"learned":[40],"log-bilinear":[42],"regression":[43],"(GloVe)":[44],"translation":[46],"(CoVe)":[47],"models,":[48],"(b)":[50],"on":[52,120,133,144],"top":[53,121],"encodings":[56],"deep":[60],"bidirectional":[61],"transformer":[62],"(BERT).":[63],"We":[64],"include":[65],"domain":[67],"adversarial":[68],"training":[69,72],"module":[70],"when":[71],"outof-domain":[74],"samples.":[75,100],"Our":[76,124],"experiments":[77],"show":[78,93],"that":[79,94],"while":[80],"BERT":[81,109],"performs":[82],"exceptionally":[83],"well":[84],"in-domain":[86],"samples,":[87],"several":[88],"runs":[89],"model":[92],"it":[95],"unstable":[97],"out-ofdomain":[99],"The":[101],"problem":[102],"mitigated":[104],"tremendously":[105],"by":[106],"(1)":[107],"combining":[108],"with":[110,130],"non-BERT":[112],"encoder,":[113],"(2)":[115],"an":[117,140],"RNNbased":[118],"classifier":[119],"BERT.":[123],"final":[125],"models":[126],"obtained":[127],"second":[128],"place":[129],"77.78%":[131],"F-Score":[132,141],"Subtask":[134,145],"A":[135],"(i.e.":[136,147],"in-domain)":[137],"achieved":[139],"79.59%":[143],"B":[146],"out-of-domain),":[148],"even":[149],"without":[150],"any":[152],"additional":[153],"external":[154],"data.":[155]},"counts_by_year":[{"year":2021,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}