{"id":"https://openalex.org/W4414891626","doi":"https://doi.org/10.3390/computers14100427","title":"LLMs for Commit Messages: A Survey and an Agent-Based Evaluation Protocol on CommitBench","display_name":"LLMs for Commit Messages: A Survey and an Agent-Based Evaluation Protocol on CommitBench","publication_year":2025,"publication_date":"2025-10-07","ids":{"openalex":"https://openalex.org/W4414891626","doi":"https://doi.org/10.3390/computers14100427"},"language":"en","primary_location":{"id":"doi:10.3390/computers14100427","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14100427","pdf_url":"https://www.mdpi.com/2073-431X/14/10/427/pdf?version=1759841214","source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2073-431X/14/10/427/pdf?version=1759841214","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022441661","display_name":"Mohamed Mehdi Trigui","orcid":"https://orcid.org/0009-0001-5706-895X"},"institutions":[{"id":"https://openalex.org/I134085113","display_name":"King Fahd University of Petroleum and Minerals","ror":"https://ror.org/03yez3163","country_code":"SA","type":"education","lineage":["https://openalex.org/I134085113"]}],"countries":["SA"],"is_corresponding":true,"raw_author_name":"Mohamed Mehdi Trigui","raw_affiliation_strings":["Information & Computer Science Department (ICS), King Fahd University of Petroleum & Minerals (KFUPM), Dhahran 31261, Saudi Arabia"],"raw_orcid":"https://orcid.org/0009-0001-5706-895X","affiliations":[{"raw_affiliation_string":"Information & Computer Science Department (ICS), King Fahd University of Petroleum & Minerals (KFUPM), Dhahran 31261, Saudi Arabia","institution_ids":["https://openalex.org/I134085113"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062511117","display_name":"Wasfi G. Al-Khatib","orcid":"https://orcid.org/0000-0002-6760-3506"},"institutions":[{"id":"https://openalex.org/I134085113","display_name":"King Fahd University of Petroleum and Minerals","ror":"https://ror.org/03yez3163","country_code":"SA","type":"education","lineage":["https://openalex.org/I134085113"]}],"countries":["SA"],"is_corresponding":true,"raw_author_name":"Wasfi G. Al-Khatib","raw_affiliation_strings":["Information & Computer Science Department (ICS), King Fahd University of Petroleum & Minerals (KFUPM), Dhahran 31261, Saudi Arabia","Interdisciplinary Research Center for Intelligent Secure Systems (IRC-ISS), King Fahd University of Petroleum & Minerals (KFUPM), Dhahran 31261, Saudi Arabia"],"raw_orcid":"https://orcid.org/0000-0002-6760-3506","affiliations":[{"raw_affiliation_string":"Information & Computer Science Department (ICS), King Fahd University of Petroleum & Minerals (KFUPM), Dhahran 31261, Saudi Arabia","institution_ids":["https://openalex.org/I134085113"]},{"raw_affiliation_string":"Interdisciplinary Research Center for Intelligent Secure Systems (IRC-ISS), King Fahd University of Petroleum & Minerals (KFUPM), Dhahran 31261, Saudi Arabia","institution_ids":["https://openalex.org/I134085113"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5022441661","https://openalex.org/A5062511117"],"corresponding_institution_ids":["https://openalex.org/I134085113"],"apc_list":{"value":1600,"currency":"CHF","value_usd":1732},"apc_paid":{"value":1600,"currency":"CHF","value_usd":1732},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.37044044,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"14","issue":"10","first_page":"427","last_page":"427"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13999","display_name":"Digital Rights Management and Security","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13999","display_name":"Digital Rights Management and Security","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12203","display_name":"Mobile Agent-Based Network Management","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10927","display_name":"Access Control and Trust","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/commit","display_name":"Commit","score":0.7558000087738037},{"id":"https://openalex.org/keywords/blueprint","display_name":"Blueprint","score":0.5547999739646912},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.5415999889373779},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.4810999929904938},{"id":"https://openalex.org/keywords/comparability","display_name":"Comparability","score":0.44839999079704285},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4171000123023987},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.3797999918460846}],"concepts":[{"id":"https://openalex.org/C153180980","wikidata":"https://www.wikidata.org/wiki/Q19776675","display_name":"Commit","level":2,"score":0.7558000087738037},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7516999840736389},{"id":"https://openalex.org/C155911762","wikidata":"https://www.wikidata.org/wiki/Q422321","display_name":"Blueprint","level":2,"score":0.5547999739646912},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.5415999889373779},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.4810999929904938},{"id":"https://openalex.org/C197947376","wikidata":"https://www.wikidata.org/wiki/Q5155608","display_name":"Comparability","level":2,"score":0.44839999079704285},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4171000123023987},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3871000111103058},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3797999918460846},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.35920000076293945},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.35120001435279846},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33559998869895935},{"id":"https://openalex.org/C63000827","wikidata":"https://www.wikidata.org/wiki/Q3080428","display_name":"Software portability","level":2,"score":0.3061999976634979},{"id":"https://openalex.org/C133112747","wikidata":"https://www.wikidata.org/wiki/Q7251931","display_name":"Protocol analysis","level":2,"score":0.3003000020980835},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3001999855041504},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.28700000047683716},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.26429998874664307},{"id":"https://openalex.org/C155951066","wikidata":"https://www.wikidata.org/wiki/Q5156254","display_name":"Compensating transaction","level":5,"score":0.2597000002861023}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/computers14100427","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14100427","pdf_url":"https://www.mdpi.com/2073-431X/14/10/427/pdf?version=1759841214","source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:46c991865f8140fa8346f3e302417663","is_oa":true,"landing_page_url":"https://doaj.org/article/46c991865f8140fa8346f3e302417663","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computers, Vol 14, Iss 10, p 427 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/computers14100427","is_oa":true,"landing_page_url":"https://doi.org/10.3390/computers14100427","pdf_url":"https://www.mdpi.com/2073-431X/14/10/427/pdf?version=1759841214","source":{"id":"https://openalex.org/S4210228075","display_name":"Computers","issn_l":"2073-431X","issn":["2073-431X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computers","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322323","display_name":"King Fahd University of Petroleum and Minerals","ror":"https://ror.org/03yez3163"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414891626.pdf","grobid_xml":"https://content.openalex.org/works/W4414891626.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2888312537","https://openalex.org/W2964322208","https://openalex.org/W3161120529","https://openalex.org/W4220877452","https://openalex.org/W4360603259","https://openalex.org/W4384154619","https://openalex.org/W4384302768","https://openalex.org/W4391741735","https://openalex.org/W4400583028","https://openalex.org/W4400680844","https://openalex.org/W4400681060","https://openalex.org/W4400768796","https://openalex.org/W4402665833","https://openalex.org/W4403210636","https://openalex.org/W4403295755","https://openalex.org/W4405937765","https://openalex.org/W4406288119","https://openalex.org/W4406324010","https://openalex.org/W4407129301","https://openalex.org/W4409217989","https://openalex.org/W4410917697","https://openalex.org/W4411659439","https://openalex.org/W4411950667","https://openalex.org/W4412047804","https://openalex.org/W4413360326","https://openalex.org/W4414304982"],"related_works":[],"abstract_inverted_index":{"Commit":[0],"messages":[1],"are":[2],"vital":[3],"for":[4],"traceability,":[5],"maintenance,":[6],"and":[7,38,64,67,92,100,103,107,110,117,134,150],"onboarding":[8],"in":[9,152],"modern":[10],"software":[11],"projects,":[12],"yet":[13],"their":[14],"quality":[15],"is":[16,145],"frequently":[17],"inconsistent.":[18],"Recent":[19],"large":[20],"language":[21,30],"models":[22],"(LLMs)":[23],"can":[24],"transform":[25],"code":[26],"diffs":[27],"into":[28],"natural":[29],"summaries,":[31],"offering":[32],"a":[33,50,71,84],"path":[34],"to":[35,125,147],"more":[36],"consistent":[37],"informative":[39],"commit":[40,55,118],"messages.":[41],"This":[42],"paper":[43],"makes":[44],"two":[45],"contributions:":[46],"(i)":[47],"it":[48,69],"provides":[49],"systematic":[51],"survey":[52],"of":[53],"automated":[54],"message":[56],"generation":[57],"with":[58,121],"LLMs,":[59],"critically":[60],"comparing":[61],"prompt-only,":[62],"fine-tuned,":[63],"retrieval-augmented":[65],"approaches;":[66],"(ii)":[68],"specifies":[70],"transparent,":[72],"agent-based":[73],"evaluation":[74,90],"blueprint":[75,144],"centered":[76],"on":[77],"CommitBench.":[78],"Unlike":[79],"prior":[80],"reviews,":[81],"we":[82],"include":[83],"detailed":[85],"dataset":[86,98],"audit,":[87],"preprocessing":[88],"impacts,":[89],"metrics,":[91],"error":[93,123],"taxonomy.":[94],"The":[95,143],"protocol":[96],"defines":[97],"usage":[99],"splits,":[101],"prompting":[102],"context":[104],"settings,":[105],"scoring":[106],"selection":[108],"rules,":[109],"reporting":[111],"guidelines":[112],"(results":[113],"by":[114],"project,":[115],"language,":[116],"type),":[119],"along":[120],"an":[122],"taxonomy":[124],"guide":[126],"qualitative":[127],"analysis.":[128],"Importantly,":[129],"this":[130],"work":[131],"emphasizes":[132],"methodology":[133],"design":[135],"rather":[136],"than":[137],"presenting":[138],"new":[139],"empirical":[140],"benchmarking":[141],"results.":[142],"intended":[146],"support":[148],"reproducibility":[149],"comparability":[151],"future":[153],"studies.":[154]},"counts_by_year":[],"updated_date":"2026-06-15T08:34:33.830935","created_date":"2025-10-10T00:00:00"}