{"id":"https://openalex.org/W4386883627","doi":"https://doi.org/10.1109/icmcis59922.2023.10253491","title":"Comparison of Transformer Models for Performance on Domain Specific Texts: A Systematic Evaluation of Intrinsic Model Performance","display_name":"Comparison of Transformer Models for Performance on Domain Specific Texts: A Systematic Evaluation of Intrinsic Model Performance","publication_year":2023,"publication_date":"2023-05-16","ids":{"openalex":"https://openalex.org/W4386883627","doi":"https://doi.org/10.1109/icmcis59922.2023.10253491"},"language":"en","primary_location":{"id":"doi:10.1109/icmcis59922.2023.10253491","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icmcis59922.2023.10253491","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 International Conference on Military Communications and Information Systems (ICMCIS)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057505929","display_name":"Giavid Valiyev","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Giavid Valiyev","raw_affiliation_strings":["Chief Technology Office","NATO Communications and Information Agency, Den Haag, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chief Technology Office","institution_ids":[]},{"raw_affiliation_string":"NATO Communications and Information Agency, Den Haag, Netherlands","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017944583","display_name":"Philip T. Eles","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Philip Eles","raw_affiliation_strings":["Operational Analysis Centre","NATO Communications and Information Agency, Den Haag, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Operational Analysis Centre","institution_ids":[]},{"raw_affiliation_string":"NATO Communications and Information Agency, Den Haag, Netherlands","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026364979","display_name":"Arvid Kok","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Arvid Kok","raw_affiliation_strings":["Chief Technology Office","NATO Communications and Information Agency, Den Haag, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chief Technology Office","institution_ids":[]},{"raw_affiliation_string":"NATO Communications and Information Agency, Den Haag, Netherlands","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037532833","display_name":"Riccardo D\u2019Ercole","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Riccardo D'Ercole","raw_affiliation_strings":["Chief Technology Office","NATO Communications and Information Agency, Den Haag, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chief Technology Office","institution_ids":[]},{"raw_affiliation_string":"NATO Communications and Information Agency, Den Haag, Netherlands","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2178,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.45027026,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9706000089645386,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.7765951156616211},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7748172283172607},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6250235438346863},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6240874528884888},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6113817095756531},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.5023150444030762},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5022273063659668},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36128026247024536}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7765951156616211},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7748172283172607},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6250235438346863},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6240874528884888},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6113817095756531},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.5023150444030762},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5022273063659668},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36128026247024536},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icmcis59922.2023.10253491","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icmcis59922.2023.10253491","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 International Conference on Military Communications and Information Systems (ICMCIS)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2294798173","https://openalex.org/W2896457183","https://openalex.org/W2965373594","https://openalex.org/W2970597249","https://openalex.org/W2970771982","https://openalex.org/W2979826702","https://openalex.org/W2980282514","https://openalex.org/W3034999214","https://openalex.org/W3092557781","https://openalex.org/W3174102480","https://openalex.org/W3183518418","https://openalex.org/W3196813608","https://openalex.org/W4240722105","https://openalex.org/W4281740565","https://openalex.org/W4287025617","https://openalex.org/W4288026527","https://openalex.org/W4288089799","https://openalex.org/W4292779060","https://openalex.org/W4385245566"],"related_works":["https://openalex.org/W2597655663","https://openalex.org/W4281476908","https://openalex.org/W4288365749","https://openalex.org/W2936497627","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W3098382480","https://openalex.org/W4287598411","https://openalex.org/W3094871513","https://openalex.org/W3100913109"],"abstract_inverted_index":{"This":[0,224],"paper":[1],"presents":[2],"a":[3,35,40,178,220],"systematic":[4],"evaluation":[5],"of":[6,20,55,61,82,89,150,161,177,195],"transformer":[7,36,63,163,200],"models":[8,64,127,164,201],"for":[9,39,110,180,202,219],"Natural":[10],"Language":[11],"Processing":[12],"(NLP)":[13],"tasks":[14,183,204],"on":[15,116,124,141,165,205],"domain-specific":[16,111,166,206],"text.":[17,167],"The":[18,148],"objective":[19],"the":[21,79,87,94,126,132,139,143,151,156,175,189,193,196,230,240,249],"study":[22,152,190],"is":[23,38,51],"to":[24,98,173,192,210,217],"understand":[25],"if":[26],"intrinsic":[27,80,107,159],"performance":[28,50,81,109,160],"statistics":[29],"can":[30,170],"help":[31],"deter-mine":[32],"how":[33,209],"suitable":[34],"model":[37,108,144,179,216],"given":[41,184],"target":[42],"corpus":[43],"and":[44,72,158,208,233],"NLP":[45,102,182,203],"task,":[46],"where":[47],"directly":[48],"measuring":[49,78],"not":[52],"possible":[53],"because":[54],"lacking":[56],"labeled":[57],"data.":[58],"A":[59],"number":[60],"pre-trained":[62],"are":[65,75],"considered,":[66],"including":[67],"BERT,":[68],"BART,":[69],"SciBERT,":[70],"RoBERTa,":[71],"XLNET":[73],"which":[74,125,142,215],"evaluated":[76],"by":[77,85,239],"such":[83],"models,":[84],"examining":[86],"distribution":[88],"sentence":[90],"embedding":[91],"vectors":[92],"in":[93,131,252],"sematic":[95],"hyperspace,":[96],"prior":[97],"applying":[99],"any":[100],"down-stream":[101],"task.":[103],"We":[104],"consider":[105],"this":[106],"texts":[112,119,123],"(in":[113],"our":[114],"case":[115,133],"military":[117],"do-main":[118],"as":[120,122],"well":[121],"were":[128],"originally":[129,146,227],"trained)":[130],"when":[134],"that":[135],"text":[136],"deviates":[137],"from":[138],"domain":[140],"was":[145,226],"trained.":[147],"results":[149,169],"provide":[153],"insights":[154],"into":[155],"characteristics":[157],"different":[162,199],"Our":[168],"be":[171],"used":[172],"inform":[174],"selection":[176],"downstream":[181],"an":[185],"input":[186,222],"dataset.":[187,223],"Overall,":[188],"contributes":[191],"understanding":[194],"trade-offs":[197],"between":[198],"texts,":[207],"make":[211],"informed":[212],"decisions":[213],"about":[214],"use":[218],"specific":[221],"pa-per":[225],"presented":[228],"at":[229],"NATO":[231],"Science":[232],"Tech-nology":[234],"Organization":[235],"Symposium":[236],"(ICMCIS)":[237],"organized":[238],"Information":[241],"Systems":[242],"Technology":[243],"(IST)":[244],"Panel,":[245],"IST-200":[246],"RSY":[247],"\u2014":[248],"ICMCIS,":[250],"held":[251],"Skopje,":[253],"North":[254],"Macedonia,":[255],"16-17":[256],"May":[257],"2023.":[258]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}