{"id":"https://openalex.org/W4414564554","doi":"https://doi.org/10.1007/s10664-025-10704-3","title":"An empirical study of large language models for type and call graph analysis in Python and JavaScript","display_name":"An empirical study of large language models for type and call graph analysis in Python and JavaScript","publication_year":2025,"publication_date":"2025-09-27","ids":{"openalex":"https://openalex.org/W4414564554","doi":"https://doi.org/10.1007/s10664-025-10704-3"},"language":"en","primary_location":{"id":"doi:10.1007/s10664-025-10704-3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-025-10704-3","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-025-10704-3.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10664-025-10704-3.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085226105","display_name":"Ashwin Prasad Shivarpatna Venkatesh","orcid":"https://orcid.org/0000-0001-6724-7962"},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Ashwin Prasad Shivarpatna Venkatesh","raw_affiliation_strings":["Heinz Nixdorf Institut, Paderborn University, Paderborn, Germany"],"raw_orcid":"https://orcid.org/0000-0001-6724-7962","affiliations":[{"raw_affiliation_string":"Heinz Nixdorf Institut, Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114427673","display_name":"Rose Sunil","orcid":null},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Rose Sunil","raw_affiliation_strings":["Paderborn University, Paderborn, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093618395","display_name":"Samkutty Sabu","orcid":"https://orcid.org/0009-0009-8868-3558"},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Samkutty Sabu","raw_affiliation_strings":["Paderborn University, Paderborn, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023740668","display_name":"Amir M. Mir","orcid":"https://orcid.org/0000-0002-9818-7752"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Amir M. Mir","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045058755","display_name":"Sofia Reis","orcid":"https://orcid.org/0000-0002-5690-2279"},"institutions":[{"id":"https://openalex.org/I121345201","display_name":"Instituto de Engenharia de Sistemas e Computadores Investiga\u00e7\u00e3o e Desenvolvimento","ror":"https://ror.org/04mqy3p58","country_code":"PT","type":"nonprofit","lineage":["https://openalex.org/I121345201","https://openalex.org/I4210125590"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Sofia Reis","raw_affiliation_strings":["IST, University of Lisbon & INESC-ID, Lisbon, Portugal"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IST, University of Lisbon & INESC-ID, Lisbon, Portugal","institution_ids":["https://openalex.org/I121345201"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076678278","display_name":"Eric Bodden","orcid":"https://orcid.org/0000-0003-3470-3647"},"institutions":[{"id":"https://openalex.org/I206945453","display_name":"Paderborn University","ror":"https://ror.org/058kzsd48","country_code":"DE","type":"education","lineage":["https://openalex.org/I206945453"]},{"id":"https://openalex.org/I4210093498","display_name":"Fraunhofer Institute for Mechatronic Systems Design","ror":"https://ror.org/004nttc42","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210093498","https://openalex.org/I4923324"]},{"id":"https://openalex.org/I4210152224","display_name":"Heinz Nixdorf Stiftung","ror":"https://ror.org/04j2tkk21","country_code":"DE","type":"nonprofit","lineage":["https://openalex.org/I4210152224"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Eric Bodden","raw_affiliation_strings":["Heinz Nixdorf Institut & Fraunhofer IEM, Paderborn University, Paderborn, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Heinz Nixdorf Institut & Fraunhofer IEM, Paderborn University, Paderborn, Germany","institution_ids":["https://openalex.org/I206945453","https://openalex.org/I4210093498","https://openalex.org/I4210152224"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5085226105"],"corresponding_institution_ids":["https://openalex.org/I206945453"],"apc_list":{"value":2290,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2290,"currency":"EUR","value_usd":2890},"fwci":4.0534,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.9462259,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"30","issue":"6","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9896000027656555,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.7436000108718872},{"id":"https://openalex.org/keywords/soundness","display_name":"Soundness","score":0.7031000256538391},{"id":"https://openalex.org/keywords/type-inference","display_name":"Type inference","score":0.6481999754905701},{"id":"https://openalex.org/keywords/javascript","display_name":"JavaScript","score":0.5594000220298767},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5461000204086304},{"id":"https://openalex.org/keywords/static-analysis","display_name":"Static analysis","score":0.49320000410079956},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4361000061035156},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.4180999994277954}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7630000114440918},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.7436000108718872},{"id":"https://openalex.org/C39920170","wikidata":"https://www.wikidata.org/wiki/Q693083","display_name":"Soundness","level":2,"score":0.7031000256538391},{"id":"https://openalex.org/C198370458","wikidata":"https://www.wikidata.org/wiki/Q586459","display_name":"Type inference","level":3,"score":0.6481999754905701},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.5810999870300293},{"id":"https://openalex.org/C544833334","wikidata":"https://www.wikidata.org/wiki/Q2005","display_name":"JavaScript","level":2,"score":0.5594000220298767},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5461000204086304},{"id":"https://openalex.org/C97686452","wikidata":"https://www.wikidata.org/wiki/Q7604153","display_name":"Static analysis","level":2,"score":0.49320000410079956},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4361000061035156},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.4180999994277954},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.38960000872612},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.387800008058548},{"id":"https://openalex.org/C91071405","wikidata":"https://www.wikidata.org/wiki/Q1413145","display_name":"Program slicing","level":3,"score":0.3734000027179718},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36550000309944153},{"id":"https://openalex.org/C198240166","wikidata":"https://www.wikidata.org/wiki/Q2298909","display_name":"Unobtrusive JavaScript","level":4,"score":0.3418000042438507},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33390000462532043},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3028999865055084},{"id":"https://openalex.org/C109701466","wikidata":"https://www.wikidata.org/wiki/Q460584","display_name":"Scala","level":3,"score":0.2985000014305115},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.28189998865127563},{"id":"https://openalex.org/C27390822","wikidata":"https://www.wikidata.org/wiki/Q320724","display_name":"Dependence analysis","level":2,"score":0.2799000144004822},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.2712000012397766},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s10664-025-10704-3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-025-10704-3","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-025-10704-3.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s10664-025-10704-3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-025-10704-3","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-025-10704-3.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320324816","display_name":"Universit\u00e4t Paderborn","ror":"https://ror.org/058kzsd48"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414564554.pdf","grobid_xml":"https://content.openalex.org/works/W4414564554.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W202191487","https://openalex.org/W2899384793","https://openalex.org/W2979826702","https://openalex.org/W3018033251","https://openalex.org/W3100869085","https://openalex.org/W3105735055","https://openalex.org/W3161685529","https://openalex.org/W4247387602","https://openalex.org/W4284685949","https://openalex.org/W4284687350","https://openalex.org/W4288089799","https://openalex.org/W4308643048","https://openalex.org/W4313130144","https://openalex.org/W4323891822","https://openalex.org/W4376167049","https://openalex.org/W4376606571","https://openalex.org/W4378509449","https://openalex.org/W4385947658","https://openalex.org/W4386081573","https://openalex.org/W4386114236","https://openalex.org/W4389161785","https://openalex.org/W4390306128","https://openalex.org/W4390437751","https://openalex.org/W4390699383","https://openalex.org/W4392414327","https://openalex.org/W4392539602","https://openalex.org/W4399567214","https://openalex.org/W4399850773","https://openalex.org/W4400485071","https://openalex.org/W4401520495","https://openalex.org/W4402410226","https://openalex.org/W4402683730","https://openalex.org/W4404990243","https://openalex.org/W4405200125","https://openalex.org/W4408146288","https://openalex.org/W4410076892","https://openalex.org/W6966882055"],"related_works":[],"abstract_inverted_index":{"Abstract":[0],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"are":[5],"increasingly":[6],"being":[7],"explored":[8],"for":[9,37,71,88,99,130,134,171,203,211],"their":[10,195,221],"potential":[11,26],"in":[12,16,74,116,155,168,192,197],"software":[13],"engineering,":[14],"particularly":[15],"static":[17,117,124,215],"analysis":[18,33,118,125,157,199,216],"tasks.":[19,119],"In":[20,161],"this":[21],"study,":[22],"we":[23,65,91],"investigate":[24],"the":[25,201],"of":[27,114],"current":[28,224],"LLMs":[29,115,163,189,213],"to":[30,84],"enhance":[31],"call-graph":[32,101,121,156,198],"and":[34,39,51,56,60,94,132,144,153,178,223],"type":[35,72,86,169,193],"inference":[36,73,170],"Python":[38,131],"JavaScript":[40,135],"programs.":[41],"We":[42],"empirically":[43],"evaluated":[44],"24":[45],"LLMs,":[46],"including":[47],"OpenAI\u2019s":[48],"GPT":[49],"series":[50],"open-source":[52],"models":[53,141],"like":[54,142,176],"LLaMA":[55],"Mistral,":[57],"using":[58],"existing":[59],"newly":[61],"developed":[62],"benchmarks.":[63],"Specifically,":[64],"enhanced":[66],"TypeEvalPy,":[67],"a":[68,111,165,209],"micro-benchmarking":[69],"framework":[70],"Python,":[75,172],"with":[76,151],"auto-generation":[77],"capabilities,":[78],"expanding":[79],"its":[80],"scope":[81],"from":[82],"860":[83],"77,268":[85],"annotations":[87],"Python.":[89],"Additionally,":[90],"introduce":[92],"SWARM-CG":[93],"SWARM-JS,":[95],"comprehensive":[96],"benchmarking":[97],"suites":[98],"evaluating":[100],"construction":[102],"tools":[103,126,175],"across":[104,158],"multiple":[105],"programming":[106],"languages.":[107,160],"Our":[108,206],"findings":[109],"reveal":[110],"contrasting":[112],"performance":[113],"For":[120],"generation,":[122],"traditional":[123,174],"such":[127,181],"as":[128,182],"PyCG":[129],"Jelly":[133],"consistently":[136],"outperform":[137],"LLMs.":[138],"While":[139],"advanced":[140],"mistral-large-it-2407-123b":[143],"gpt-4o":[145],"show":[146],"promise,":[147],"they":[148],"still":[149],"struggle":[150],"completeness":[152],"soundness":[154],"both":[159],"contrast,":[162],"demonstrate":[164],"clear":[166],"advantage":[167],"surpassing":[173],"HeaderGen":[177],"hybrid":[179],"approaches":[180],"HiTyper.":[183],"These":[184],"results":[185],"suggest":[186],"that,":[187],"while":[188],"hold":[190],"promise":[191],"inference,":[194],"limitations":[196],"highlight":[200],"need":[202],"further":[204],"research.":[205],"study":[207],"provides":[208],"foundation":[210],"integrating":[212],"into":[214,220],"workflows,":[217],"offering":[218],"insights":[219],"strengths":[222],"limitations.":[225]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-06-13T06:13:01.061226","created_date":"2025-10-10T00:00:00"}