{"id":"https://openalex.org/W4411950661","doi":"https://doi.org/10.1109/forge66646.2025.00027","title":"Enhancing Pull Request Reviews: Leveraging Large Language Models to Detect Inconsistencies Between Issues and Pull Requests","display_name":"Enhancing Pull Request Reviews: Leveraging Large Language Models to Detect Inconsistencies Between Issues and Pull Requests","publication_year":2025,"publication_date":"2025-04-27","ids":{"openalex":"https://openalex.org/W4411950661","doi":"https://doi.org/10.1109/forge66646.2025.00027"},"language":"en","primary_location":{"id":"doi:10.1109/forge66646.2025.00027","is_oa":false,"landing_page_url":"https://doi.org/10.1109/forge66646.2025.00027","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACM Second International Conference on AI Foundation Models and Software Engineering (Forge)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5118792446","display_name":"Ali Tunahan I\u015f\u0131k","orcid":null},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":true,"raw_author_name":"Ali Tunahan I\u015f\u0131k","raw_affiliation_strings":["Bilkent University,Computer Science,Ankara,Turkey"],"affiliations":[{"raw_affiliation_string":"Bilkent University,Computer Science,Ankara,Turkey","institution_ids":["https://openalex.org/I168864056"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118792447","display_name":"Hatice K\u00fcbra \u00c7a\u011flar","orcid":null},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Hatice K\u00fcbra \u00c7a\u011flar","raw_affiliation_strings":["Bilkent University,Computer Science,Ankara,Turkey"],"affiliations":[{"raw_affiliation_string":"Bilkent University,Computer Science,Ankara,Turkey","institution_ids":["https://openalex.org/I168864056"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027949836","display_name":"Eray T\u00fcz\u00fcn","orcid":"https://orcid.org/0000-0002-5550-7816"},"institutions":[{"id":"https://openalex.org/I168864056","display_name":"Bilkent University","ror":"https://ror.org/02vh8a032","country_code":"TR","type":"education","lineage":["https://openalex.org/I168864056"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Eray T\u00fcz\u00fcn","raw_affiliation_strings":["Bilkent University,Computer Science,Ankara,Turkey"],"affiliations":[{"raw_affiliation_string":"Bilkent University,Computer Science,Ankara,Turkey","institution_ids":["https://openalex.org/I168864056"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5118792446"],"corresponding_institution_ids":["https://openalex.org/I168864056"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.08563856,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"168","last_page":"178"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9193000197410583,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9193000197410583,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.796099066734314}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.796099066734314}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/forge66646.2025.00027","is_oa":false,"landing_page_url":"https://doi.org/10.1109/forge66646.2025.00027","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACM Second International Conference on AI Foundation Models and Software Engineering (Forge)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W2053154970","https://openalex.org/W2097227214","https://openalex.org/W2137444776","https://openalex.org/W2139885493","https://openalex.org/W2758152813","https://openalex.org/W2901222374","https://openalex.org/W2999118008","https://openalex.org/W3006568433","https://openalex.org/W3102072454","https://openalex.org/W3154383848","https://openalex.org/W3186179984","https://openalex.org/W3206481103","https://openalex.org/W4281702414","https://openalex.org/W4294214983","https://openalex.org/W4295953690","https://openalex.org/W4297254899","https://openalex.org/W4301862456","https://openalex.org/W4308643029","https://openalex.org/W4312864407","https://openalex.org/W4313229660","https://openalex.org/W4399213445","https://openalex.org/W4399577186","https://openalex.org/W4400681060","https://openalex.org/W4405602296","https://openalex.org/W6769243733","https://openalex.org/W6851775633","https://openalex.org/W6861347535"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Context:":[0],"Efficient":[1],"Pull":[2],"Request":[3],"(PR)":[4],"review":[5,28,68,258,366],"process":[6,13,69,191],"is":[7,344],"critical":[8],"in":[9,41,48,57,83,192,337],"software":[10],"development.":[11],"This":[12,50,123,346],"includes":[14],"checking":[15],"the":[16,35,44,58,66,96,128,132,178,193,198,255,263,283,290,294,332,348,364],"alignment":[17,102,280],"between":[18,34,77,227,308],"PRs":[19,94,238,320],"and":[20,43,80,121,142,160,169,172,186,214,219,231,273,287,301,312,357,371],"their":[21,101,173,184,341,355],"corresponding":[22,104],"issues.":[23],"The":[24,145,188],"traditional":[25],"manual":[26,189,369],"PR":[27,59,67,108,228,271,274,309,365],"often":[29],"struggles":[30],"with":[31,103,166,234],"identifying":[32,167],"inconsistencies":[33,56,76],"intended":[36],"improvements":[37],"or":[38,119,328],"fixes":[39],"outlined":[40],"issues":[42,159],"actual":[45],"changes":[46,82],"proposed":[47],"PRs.":[49,161,248],"difference":[51],"can":[52,321],"lead":[53],"to":[54,64,74,99,154,182,243,353],"overlooked":[55],"acceptance":[60],"process.Objective:":[61],"We":[62,86],"aim":[63],"enhance":[65,354],"by":[70],"leveraging":[71],"modern":[72],"LLMs":[73,334],"detect":[75],"issue":[78,105,269],"descriptions":[79],"code":[81,257,373],"submitted":[84],"PRs.Method:":[85],"manually":[87,179],"labeled":[88,124,180],"a":[89,305],"statistically":[90],"significant":[91],"sample":[92],"of":[93,113,134,158,201,236,246,279,299],"from":[95],"Transformers":[97,195],"repository":[98,196],"assess":[100,183],"descriptions.":[106],"Each":[107,162],"was":[109,164,225],"categorized":[110],"into":[111],"one":[112],"four":[114,135],"groups:":[115],"exact,":[116,207,241],"missing,":[117,210],"tangling,":[118,213],"missing":[120,170,218,327],"tangling.":[122],"dataset":[125],"served":[126],"as":[127,240,326],"benchmark":[129],"for":[130,253,350],"evaluating":[131],"performance":[133,343],"widely":[136],"used":[137],"models:":[138],"Llama3.1-70B-Instruct,":[139],"Llama-3.1-405B-Instruct,":[140],"GPT-4o,":[141],"GPT-4o":[143,286],"mini.":[144],"models":[146,284],"were":[147,175,206,209,212],"tested":[148,333],"using":[149],"three":[150],"distinct":[151],"prompts":[152],"designed":[153],"capture":[155],"different":[156],"aspects":[157],"model":[163],"tasked":[165],"tangled":[168],"elements,":[171],"outputs":[174],"compared":[176,242],"against":[177],"data":[181],"accuracy":[185,356],"reliability.Results:":[187],"labeling":[190],"stratified-sampled":[194],"revealed":[197,317],"following":[199],"distribution":[200],"PR-issue":[202,339],"pair":[203],"alignments:":[204],"68.04%":[205],"16.5%":[208],"13.40%":[211],"2.06%":[215],"exhibited":[216],"both":[217],"tangling":[220,329],"characteristics.":[221],"A":[222],"strong":[223],"correlation":[224,307],"observed":[226],"merge":[229,310],"status":[230,311],"exact":[232,313],"alignment,":[233,314,340],"75.46%":[235],"merged":[237,319],"classified":[239],"only":[244],"29.03%":[245],"unmerged":[247],"These":[249],"findings":[250],"highlight":[251],"opportunities":[252],"improving":[254],"current":[256,342],"process.":[259],"For":[260],"automated":[261],"classification,":[262],"most":[264],"effective":[265],"prompt":[266],"configuration":[267],"combined":[268],"text,":[270,272],"diff,":[275],"enabling":[276],"better":[277],"detection":[278],"inconsistencies.":[281],"Among":[282],"tested,":[285],"Llama-3.1-405B-Instruct":[288],"delivered":[289],"highest":[291],"performance,":[292],"achieving":[293],"best":[295],"F1":[296],"weighted":[297],"scores":[298],"0.5948":[300],"0.6190,":[302],"respectively.Conclusion:":[303],"Despite":[304],"notable":[306],"our":[315],"analysis":[316],"that":[318],"still":[322],"contain":[323],"inconsistencies,":[324],"such":[325],"changes.":[330],"While":[331],"showed":[335],"potential":[336],"automating":[338],"limited.":[345],"underscores":[347],"need":[349],"further":[351],"refinement":[352],"reliability.":[358],"Improved":[359],"LLM-based":[360],"tools":[361],"could":[362],"streamline":[363],"process,":[367],"reducing":[368],"effort":[370],"enhancing":[372],"quality.":[374]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}