{"id":"https://openalex.org/W4327503230","doi":"https://doi.org/10.1109/tc.2023.3257518","title":"Improving Log-Based Anomaly Detection by Pre-Training Hierarchical Transformers","display_name":"Improving Log-Based Anomaly Detection by Pre-Training Hierarchical Transformers","publication_year":2023,"publication_date":"2023-03-15","ids":{"openalex":"https://openalex.org/W4327503230","doi":"https://doi.org/10.1109/tc.2023.3257518"},"language":"en","primary_location":{"id":"doi:10.1109/tc.2023.3257518","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tc.2023.3257518","pdf_url":null,"source":{"id":"https://openalex.org/S157670870","display_name":"IEEE Transactions on Computers","issn_l":"0018-9340","issn":["0018-9340","1557-9956","2326-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Computers","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061624006","display_name":"Shaohan Huang","orcid":"https://orcid.org/0000-0003-4324-6337"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Shaohan Huang","raw_affiliation_strings":["Sino-German Joint Software Institute, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Sino-German Joint Software Institute, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100330624","display_name":"Yi Liu","orcid":"https://orcid.org/0000-0003-1829-2817"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Liu","raw_affiliation_strings":["Sino-German Joint Software Institute, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Sino-German Joint Software Institute, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001203568","display_name":"Carol Fung","orcid":null},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Carol Fung","raw_affiliation_strings":["Concordia Institude for Information Systems Engineering, Concordia University, Montreal, QC, Canada"],"affiliations":[{"raw_affiliation_string":"Concordia Institude for Information Systems Engineering, Concordia University, Montreal, QC, Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100351649","display_name":"He Wang","orcid":"https://orcid.org/0000-0001-9389-9761"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"He Wang","raw_affiliation_strings":["Sino-German Joint Software Institute, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Sino-German Joint Software Institute, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018705589","display_name":"Hailong Yang","orcid":"https://orcid.org/0000-0003-1101-7927"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hailong Yang","raw_affiliation_strings":["Sino-German Joint Software Institute, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Sino-German Joint Software Institute, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I82880672"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074183877","display_name":"Zhongzhi Luan","orcid":"https://orcid.org/0000-0002-7186-0556"},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]},{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhongzhi Luan","raw_affiliation_strings":["Sino-German Joint Software Institute, Beihang University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Sino-German Joint Software Institute, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I82880672"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5061624006"],"corresponding_institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I82880672"],"apc_list":null,"apc_paid":null,"fwci":7.8024,"has_fulltext":false,"cited_by_count":39,"citation_normalized_percentile":{"value":0.97763667,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"72","issue":"9","first_page":"2656","last_page":"2667"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10400","display_name":"Network Security and Intrusion Detection","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7796542644500732},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6266711950302124},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5609992146492004},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5135371685028076},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49849486351013184},{"id":"https://openalex.org/keywords/parsing","display_name":"Parsing","score":0.45804423093795776},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.378221333026886},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3420056104660034},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.32978472113609314},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.09337228536605835}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7796542644500732},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6266711950302124},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5609992146492004},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5135371685028076},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49849486351013184},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.45804423093795776},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.378221333026886},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3420056104660034},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32978472113609314},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.09337228536605835},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tc.2023.3257518","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tc.2023.3257518","pdf_url":null,"source":{"id":"https://openalex.org/S157670870","display_name":"IEEE Transactions on Computers","issn_l":"0018-9340","issn":["0018-9340","1557-9956","2326-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Computers","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7200000286102295}],"awards":[{"id":"https://openalex.org/G5774836301","display_name":null,"funder_award_id":"62072018","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6322973505","display_name":null,"funder_award_id":"61732002","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W2039157918","https://openalex.org/W2064675550","https://openalex.org/W2094924503","https://openalex.org/W2105454049","https://openalex.org/W2107263349","https://openalex.org/W2114554028","https://openalex.org/W2141041105","https://openalex.org/W2147296306","https://openalex.org/W2401686019","https://openalex.org/W2525778437","https://openalex.org/W2560021099","https://openalex.org/W2754665629","https://openalex.org/W2767094836","https://openalex.org/W2896457183","https://openalex.org/W2898998129","https://openalex.org/W2911489562","https://openalex.org/W2947815220","https://openalex.org/W2952729433","https://openalex.org/W2962739339","https://openalex.org/W2965838158","https://openalex.org/W2969601108","https://openalex.org/W2970771982","https://openalex.org/W2979826702","https://openalex.org/W3094313992","https://openalex.org/W3095840026","https://openalex.org/W3183197931","https://openalex.org/W3199174176","https://openalex.org/W4385245566","https://openalex.org/W6685158001","https://openalex.org/W6712994927","https://openalex.org/W6727690538","https://openalex.org/W6739901393","https://openalex.org/W6755207826","https://openalex.org/W6766795243","https://openalex.org/W6798271659"],"related_works":["https://openalex.org/W4381248170","https://openalex.org/W3189621521","https://openalex.org/W2173794830","https://openalex.org/W1502858101","https://openalex.org/W4288365749","https://openalex.org/W2936497627","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W3098382480","https://openalex.org/W4287598411"],"abstract_inverted_index":{"Pre-trained":[0],"models,":[1,10,156],"such":[2,11],"as":[3,12,88],"BERT,":[4,13],"have":[5,14],"resulted":[6,15],"in":[7,16,19,30],"significant":[8,17],"pre-trained":[9,57,70],"improvements":[18],"many":[20],"natural":[21],"language":[22,58],"processing":[23],"(NLP)":[24],"applications.":[25],"However,":[26],"due":[27],"to":[28,40,52,101,111,151],"differences":[29],"word":[31],"distribution":[32],"and":[33,125,166,169],"domain":[34],"data":[35],"distribution,":[36],"applying":[37],"NLP":[38],"advancements":[39],"log":[41,62,71,99,113,131,144,164],"analysis":[42],"directly":[43],"faces":[44],"some":[45],"performance":[46,149],"challenges.":[47],"This":[48],"paper":[49],"studies":[50],"how":[51],"adapt":[53],"the":[54,98,161],"recently":[55],"introduced":[56],"model":[59,73,110,128],"BERT":[60,147],"for":[61,122],"analysis.":[63],"In":[64],"this":[65],"work,":[66,83],"we":[67,91],"propose":[68],"a":[69,107],"representation":[72],"with":[74,129],"hierarchical":[75,108],"bidirectional":[76],"encoder":[77],"transformers":[78,109],"(namely,":[79],"HilBERT).":[80],"Unlike":[81],"previous":[82,154],"which":[84],"used":[85],"raw":[86],"text":[87],"pre-training":[89],"data,":[90],"parse":[92],"logs":[93],"into":[94],"templates":[95,100],"before":[96],"using":[97],"pre-train":[102],"HilBERT.":[103],"We":[104,117],"also":[105],"design":[106],"capture":[112],"template":[114],"sequence-level":[115],"information.":[116],"use":[118],"log-based":[119],"anomaly":[120],"detection":[121],"downstream":[123],"tasks":[124],"fine-tune":[126],"our":[127],"different":[130],"data.":[132,145],"Our":[133],"experiments":[134],"demonstrate":[135],"that":[136,152],"HilBERT":[137,157],"outperforms":[138],"other":[139],"baseline":[140],"techniques":[141],"on":[142],"unstable":[143],"While":[146],"obtains":[148],"comparable":[150],"of":[153,163],"state-of-the-art":[155],"can":[158],"significantly":[159],"address":[160],"problem":[162],"instability":[165],"achieve":[167],"accurate":[168],"robust":[170],"results.":[171]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":3}],"updated_date":"2026-03-27T14:29:43.386196","created_date":"2025-10-10T00:00:00"}