{"id":"https://openalex.org/W4402443006","doi":"https://doi.org/10.1145/3650212.3652123","title":"A Large-Scale Evaluation for Log Parsing Techniques: How Far Are We?","display_name":"A Large-Scale Evaluation for Log Parsing Techniques: How Far Are We?","publication_year":2024,"publication_date":"2024-09-11","ids":{"openalex":"https://openalex.org/W4402443006","doi":"https://doi.org/10.1145/3650212.3652123"},"language":"en","primary_location":{"id":"doi:10.1145/3650212.3652123","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3650212.3652123","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068670072","display_name":"Zhihan Jiang","orcid":"https://orcid.org/0009-0003-1988-6219"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":true,"raw_author_name":"Zhihan Jiang","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100705518","display_name":"Jinyang Liu","orcid":"https://orcid.org/0000-0003-0037-1912"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jinyang Liu","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040653412","display_name":"Junjie Huang","orcid":"https://orcid.org/0009-0004-6962-5292"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Junjie Huang","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100426730","display_name":"Yichen Li","orcid":"https://orcid.org/0009-0009-8370-644X"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yichen Li","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080873193","display_name":"Yintong Huo","orcid":"https://orcid.org/0009-0006-8798-5667"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yintong Huo","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042468232","display_name":"Jiazhen Gu","orcid":"https://orcid.org/0000-0002-5831-9474"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jiazhen Gu","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048929944","display_name":"Zhuangbin Chen","orcid":"https://orcid.org/0000-0001-5158-6716"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhuangbin Chen","raw_affiliation_strings":["Sun Yat-sen University, Zhuhai, China"],"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Zhuhai, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048669373","display_name":"Jieming Zhu","orcid":"https://orcid.org/0000-0002-5666-8320"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jieming Zhu","raw_affiliation_strings":["Huawei Ark Lab, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Huawei Ark Lab, Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069596903","display_name":"Michael R. Lyu","orcid":"https://orcid.org/0000-0002-3666-5798"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Michael R. Lyu","raw_affiliation_strings":["Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5068670072"],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":16.2801,"has_fulltext":false,"cited_by_count":47,"citation_normalized_percentile":{"value":0.99417772,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"223","last_page":"234"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10400","display_name":"Network Security and Intrusion Detection","score":0.9753999710083008,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9653000235557556,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6795347332954407},{"id":"https://openalex.org/keywords/parsing","display_name":"Parsing","score":0.6556909084320068},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5560450553894043},{"id":"https://openalex.org/keywords/bottom-up-parsing","display_name":"Bottom-up parsing","score":0.4150667190551758},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3107658922672272},{"id":"https://openalex.org/keywords/top-down-parsing","display_name":"Top-down parsing","score":0.10237732529640198},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.08006051182746887},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.07704022526741028}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6795347332954407},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.6556909084320068},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5560450553894043},{"id":"https://openalex.org/C60690694","wikidata":"https://www.wikidata.org/wiki/Q894902","display_name":"Bottom-up parsing","level":4,"score":0.4150667190551758},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3107658922672272},{"id":"https://openalex.org/C42560504","wikidata":"https://www.wikidata.org/wiki/Q15419395","display_name":"Top-down parsing","level":3,"score":0.10237732529640198},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.08006051182746887},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07704022526741028}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3650212.3652123","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3650212.3652123","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM SIGSOFT International Symposium on Software Testing and Analysis","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W95608104","https://openalex.org/W1920048997","https://openalex.org/W2015979301","https://openalex.org/W2076809217","https://openalex.org/W2099834293","https://openalex.org/W2153470728","https://openalex.org/W2170890990","https://openalex.org/W2208211896","https://openalex.org/W2583874385","https://openalex.org/W2734941459","https://openalex.org/W2754665629","https://openalex.org/W2767094836","https://openalex.org/W2792207129","https://openalex.org/W2888491619","https://openalex.org/W2955925687","https://openalex.org/W2963999143","https://openalex.org/W2999013495","https://openalex.org/W3143573050","https://openalex.org/W3150838151","https://openalex.org/W3167387776","https://openalex.org/W3178329914","https://openalex.org/W4205171160","https://openalex.org/W4229911753","https://openalex.org/W4232648333","https://openalex.org/W4238623774","https://openalex.org/W4242838928","https://openalex.org/W4246553962","https://openalex.org/W4246788636","https://openalex.org/W4247618872","https://openalex.org/W4285490493","https://openalex.org/W4300541501","https://openalex.org/W4307510928","https://openalex.org/W4319874037","https://openalex.org/W4319990463","https://openalex.org/W4365513119","https://openalex.org/W4367146761","https://openalex.org/W4379116564","https://openalex.org/W4384211336","https://openalex.org/W4388240174","https://openalex.org/W6849072694","https://openalex.org/W6852168108"],"related_works":["https://openalex.org/W2619584063","https://openalex.org/W2952780262","https://openalex.org/W3088470625","https://openalex.org/W2142145894","https://openalex.org/W3035970863","https://openalex.org/W3143982968","https://openalex.org/W332673702","https://openalex.org/W2152451097","https://openalex.org/W2571839828","https://openalex.org/W1481527871"],"abstract_inverted_index":{"Log":[0],"data":[1,35,131,187],"have":[2],"facilitated":[3],"various":[4,48],"tasks":[5],"of":[6,22,43,117,129,143,159,182,199,238],"software":[7,134,213],"development":[8],"and":[9,15,57,78,168,236],"maintenance,":[10],"such":[11,217],"as":[12,63],"testing,":[13],"debugging":[14],"diagnosing.":[16],"Due":[17],"to":[18,29,53,85,178,185,194],"the":[19,41,127,180,192,196,234],"unstructured":[20],"nature":[21],"logs,":[23],"log":[24,31,38,44,70,89,103,119,130,146,162,200,239],"parsing":[25,216],"is":[26,97,219],"typically":[27],"required":[28],"transform":[30],"messages":[32],"into":[33],"structured":[34],"for":[36,68,83,105,212],"automated":[37],"analysis.":[39],"Given":[40],"abundance":[42],"parsers":[45,90,104,163,201,240],"that":[46,204],"employ":[47],"techniques,":[49],"evaluating":[50],"these":[51,102,110],"tools":[52],"comprehend":[54],"their":[55,246],"characteristics":[56,128],"performance":[58,198],"becomes":[59],"imperative.":[60],"Loghub":[61],"serves":[62],"a":[64,114,156,165,174,224],"commonly":[65],"used":[66],"dataset":[67],"benchmarking":[69],"parsers,":[71],"but":[72],"it":[73,222],"suffers":[74],"from":[75],"limited":[76],"scale":[77],"representativeness,":[79],"posing":[80],"significant":[81],"challenges":[82],"studies":[84],"comprehensively":[86],"evaluate":[87],"existing":[88,183],"or":[91],"develop":[92],"new":[93,115,175],"methods.":[94],"This":[95],"limitation":[96],"particularly":[98],"pronounced":[99],"when":[100],"assessing":[101],"production":[106,249],"use.":[107],"To":[108],"address":[109],"limitations,":[111],"we":[112,154,172],"provide":[113],"collection":[116],"annotated":[118],"datasets,":[120],"denoted":[121],"Loghub-2.0,":[122,153],"which":[123],"can":[124],"better":[125],"reflect":[126],"in":[132,148,164,241,248],"real-world":[133],"systems.":[135,250],"Loghub-2.0":[136],"comprises":[137],"14":[138],"datasets":[139],"with":[140],"an":[141],"average":[142],"3.6":[144],"million":[145],"lines":[147],"each":[149],"dataset.":[150],"Based":[151],"on":[152,202,233],"conduct":[155],"thorough":[157],"re-evaluation":[158],"15":[160],"state-of-the-art":[161],"more":[166],"rigorous":[167],"practical":[169,242],"setting.":[170],"Particularly,":[171],"introduce":[173],"evaluation":[176,235],"metric":[177],"mitigate":[179],"sensitivity":[181],"metrics":[184],"imbalanced":[186],"distributions.":[188],"We":[189,226],"are":[190],"also":[191],"first":[193],"investigate":[195],"granular":[197],"logs":[203,218],"represent":[205],"rare":[206],"system":[207],"events,":[208],"offering":[209],"in-depth":[210],"details":[211],"diagnosis.":[214],"Accurately":[215],"essential,":[220],"yet":[221],"remains":[223],"challenge.":[225],"believe":[227],"this":[228],"work":[229],"could":[230],"shed":[231],"light":[232],"design":[237],"settings,":[243],"thereby":[244],"facilitating":[245],"deployment":[247]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":33},{"year":2024,"cited_by_count":6}],"updated_date":"2026-04-03T22:45:19.894376","created_date":"2025-10-10T00:00:00"}