{"id":"https://openalex.org/W4410537880","doi":"https://doi.org/10.1109/icst62969.2025.10988921","title":"Benchmarking Open-Source Large Language Models for Log Level Suggestion","display_name":"Benchmarking Open-Source Large Language Models for Log Level Suggestion","publication_year":2025,"publication_date":"2025-03-31","ids":{"openalex":"https://openalex.org/W4410537880","doi":"https://doi.org/10.1109/icst62969.2025.10988921"},"language":"en","primary_location":{"id":"doi:10.1109/icst62969.2025.10988921","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst62969.2025.10988921","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114285400","display_name":"Yi Wen Heng","orcid":null},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Yi Wen Heng","raw_affiliation_strings":["Concordia University,Software PErformance, Analysis, and Reliability (SPEAR) lab,Montreal,Canada"],"affiliations":[{"raw_affiliation_string":"Concordia University,Software PErformance, Analysis, and Reliability (SPEAR) lab,Montreal,Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5095376985","display_name":"Zeyang Ma","orcid":"https://orcid.org/0000-0002-0390-1547"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zeyang Ma","raw_affiliation_strings":["Concordia University,Software PErformance, Analysis, and Reliability (SPEAR) lab,Montreal,Canada"],"affiliations":[{"raw_affiliation_string":"Concordia University,Software PErformance, Analysis, and Reliability (SPEAR) lab,Montreal,Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004382658","display_name":"Zhenhao Li","orcid":"https://orcid.org/0000-0002-4909-1535"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Zhenhao Li","raw_affiliation_strings":["York University,Toronto,Canada"],"affiliations":[{"raw_affiliation_string":"York University,Toronto,Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020218671","display_name":"Dong Jae Kim","orcid":"https://orcid.org/0000-0002-3181-0001"},"institutions":[{"id":"https://openalex.org/I118353179","display_name":"DePaul University","ror":"https://ror.org/04xtx5t16","country_code":"US","type":"education","lineage":["https://openalex.org/I118353179"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dong Jae Kim","raw_affiliation_strings":["DePaul University,Chicago,Illinois,USA"],"affiliations":[{"raw_affiliation_string":"DePaul University,Chicago,Illinois,USA","institution_ids":["https://openalex.org/I118353179"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069372430","display_name":"Tse-Hsun Chen","orcid":"https://orcid.org/0000-0003-4027-0905"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Tse-Hsun Chen","raw_affiliation_strings":["Concordia University,Software PErformance, Analysis, and Reliability (SPEAR) lab,Montreal,Canada"],"affiliations":[{"raw_affiliation_string":"Concordia University,Software PErformance, Analysis, and Reliability (SPEAR) lab,Montreal,Canada","institution_ids":["https://openalex.org/I60158472"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5114285400"],"corresponding_institution_ids":["https://openalex.org/I60158472"],"apc_list":null,"apc_paid":null,"fwci":2.8414,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.90842807,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"314","last_page":"325"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9168999791145325,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9168999791145325,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7901668548583984},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.740199863910675},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.48542818427085876},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.48345932364463806},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3983769714832306},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.09514299035072327}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7901668548583984},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.740199863910675},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.48542818427085876},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.48345932364463806},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3983769714832306},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.09514299035072327},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icst62969.2025.10988921","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst62969.2025.10988921","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W1912982817","https://openalex.org/W2027380800","https://openalex.org/W2136715810","https://openalex.org/W2531425405","https://openalex.org/W2763526563","https://openalex.org/W2786424616","https://openalex.org/W2811120218","https://openalex.org/W2888115557","https://openalex.org/W2947815220","https://openalex.org/W2953520354","https://openalex.org/W2963999143","https://openalex.org/W2966971704","https://openalex.org/W2973461969","https://openalex.org/W3004071943","https://openalex.org/W3098605233","https://openalex.org/W3132847876","https://openalex.org/W3150838151","https://openalex.org/W3162546773","https://openalex.org/W3173777717","https://openalex.org/W3177813494","https://openalex.org/W3178329914","https://openalex.org/W3203565869","https://openalex.org/W4214643715","https://openalex.org/W4284665614","https://openalex.org/W4285490493","https://openalex.org/W4286331401","https://openalex.org/W4384345673","https://openalex.org/W4384345685","https://openalex.org/W4388483112","https://openalex.org/W4389524484","https://openalex.org/W4394745194","https://openalex.org/W4394745212","https://openalex.org/W4400582976","https://openalex.org/W4402442387","https://openalex.org/W4404057519","https://openalex.org/W6683604450","https://openalex.org/W6690815549","https://openalex.org/W6740934949","https://openalex.org/W6753967410","https://openalex.org/W6772383348","https://openalex.org/W6784096203","https://openalex.org/W6803842174","https://openalex.org/W6852331649","https://openalex.org/W6854159973","https://openalex.org/W6856227841","https://openalex.org/W6856984059","https://openalex.org/W6861381487","https://openalex.org/W6881047908"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4,25],"become":[5],"a":[6,51,55,101],"focal":[7],"point":[8],"of":[9,29,54,109,117],"research":[10,173],"across":[11,158],"various":[12],"domains,":[13],"including":[14,152],"software":[15,32,60],"engineering,":[16],"where":[17],"their":[18,38,208],"capabilities":[19],"are":[20],"increasingly":[21],"leveraged.":[22],"Recent":[23],"studies":[24],"explored":[26],"the":[27,63,89,107,115,163],"integration":[28],"LLMs":[30,120,160,190],"into":[31],"development":[33],"tools":[34],"and":[35,45,111,143,155,197],"frameworks,":[36],"revealing":[37],"potential":[39,204],"to":[40,87,105,134,161,210],"enhance":[41],"performance":[42,116],"in":[43,121,207],"text":[44],"code-related":[46],"tasks.":[47],"Log":[48],"level":[49,92,123,170],"is":[50,174,201],"key":[52],"part":[53],"logging":[56,94],"statement":[57],"that":[58,70,187],"allows":[59],"developers":[61],"control":[62],"information":[64,142],"recorded":[65],"during":[66],"system":[67],"runtime.":[68],"Given":[69],"log":[71,122,169,212],"messages":[72],"often":[73],"mix":[74],"natural":[75],"language":[76,81],"with":[77,194],"code-like":[78],"variables,":[79],"LLMs'":[80],"translation":[82],"abilities":[83],"could":[84],"be":[85],"applied":[86],"determine":[88],"suitable":[90,198],"verbosity":[91],"for":[93,127,167,205],"statements.":[95],"In":[96],"this":[97],"paper,":[98],"we":[99],"undertake":[100],"detailed":[102],"empirical":[103],"analysis":[104],"investigate":[106],"impact":[108],"characteristics":[110],"learning":[112],"paradigms":[113],"on":[114,179],"12":[118],"open-source":[119,128],"suggestion.":[124],"We":[125,147],"opted":[126],"models":[129],"because":[130],"they":[131],"enable":[132],"us":[133],"utilize":[135],"in-house":[136],"code":[137],"while":[138],"effectively":[139,193],"protecting":[140],"sensitive":[141],"maintaining":[144],"data":[145],"security.":[146],"examine":[148],"several":[149],"prompting":[150],"strategies,":[151],"Zero-shot,":[153],"Few-shot,":[154],"fine-tuning":[156],"techniques,":[157,199],"different":[159],"identify":[162],"most":[164],"effective":[165],"combinations":[166],"accurate":[168],"suggestions.":[171],"Our":[172],"supported":[175],"by":[176],"experiments":[177],"conducted":[178],"9":[180],"large-scale":[181],"Java":[182],"systems.":[183],"The":[184],"results":[185],"indicate":[186],"although":[188],"smaller":[189],"can":[191],"perform":[192],"appropriate":[195],"instruction":[196],"there":[200],"still":[202],"considerable":[203],"improvement":[206],"ability":[209],"suggest":[211],"levels.":[213]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}