{"id":"https://openalex.org/W4411993048","doi":"https://doi.org/10.1016/j.jss.2025.112541","title":"LLMs and Stack Overflow discussions: Reliability, impact, and challenges","display_name":"LLMs and Stack Overflow discussions: Reliability, impact, and challenges","publication_year":2025,"publication_date":"2025-07-03","ids":{"openalex":"https://openalex.org/W4411993048","doi":"https://doi.org/10.1016/j.jss.2025.112541"},"language":"en","primary_location":{"id":"doi:10.1016/j.jss.2025.112541","doi":"https://doi.org/10.1016/j.jss.2025.112541","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2025.112541","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-b","cwts-core","dongbi-c","fnege-4","jufo-3","ki-jl-1","norway-2","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.jss.2025.112541","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001086221","display_name":"L\u00e9uson Da Silva","orcid":"https://orcid.org/0000-0002-9086-9038","observed_orcids":["https://orcid.org/0000-0002-9086-9038"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Leuson Da Silva","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-9086-9038","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090389831","display_name":"Jordan Samhi","orcid":"https://orcid.org/0000-0001-6052-6184","observed_orcids":["https://orcid.org/0000-0001-6052-6184"]},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jordan Samhi","raw_affiliation_strings":["CISPA, Saarbr\u00fccken, Germany"],"raw_orcid":"https://orcid.org/0000-0001-6052-6184","affiliations":[{"raw_affiliation_string":"CISPA, Saarbr\u00fccken, Germany","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071052367","display_name":"Foutse Khomh","orcid":"https://orcid.org/0000-0002-5704-4173","observed_orcids":["https://orcid.org/0000-0002-5704-4173"]},"institutions":[{"id":"https://openalex.org/I45683168","display_name":"Polytechnique Montr\u00e9al","ror":"https://ror.org/05f8d4e86","country_code":"CA","type":"education","lineage":["https://openalex.org/I45683168"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Foutse Khomh","raw_affiliation_strings":["Polytechnique Montreal, Montreal, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Polytechnique Montreal, Montreal, Canada","institution_ids":["https://openalex.org/I45683168"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5001086221"],"corresponding_institution_ids":[],"apc_list":{"value":3820,"currency":"USD","value_usd":3820},"apc_paid":{"value":3820,"currency":"USD","value_usd":3820},"fwci":18.6722,"has_fulltext":true,"cited_by_count":38,"citation_normalized_percentile":{"value":0.99386033,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"230","issue":null,"first_page":"112541","last_page":"112541"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.7289999723434448,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.7289999723434448,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.259799987077713,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0017000000225380063,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9789999723434448},{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.9210000038146973},{"id":"https://openalex.org/keywords/llama","display_name":"llamas","score":0.9900000095367432},{"id":"https://openalex.org/keywords/stack-overflow","display_name":"Stack Overflow","score":0.9909999966621399},{"id":"https://openalex.org/keywords/programming-question-answering","display_name":"programming question answering","score":0.7089999914169312},{"id":"https://openalex.org/keywords/generative-ai","display_name":"generative AI","score":0.6990000009536743},{"id":"https://openalex.org/keywords/answer-reliability","display_name":"answer reliability","score":0.7310000061988831},{"id":"https://openalex.org/keywords/llm-comparison","display_name":"LLM comparison","score":0.7080000042915344},{"id":"https://openalex.org/keywords/human-expertise","display_name":"human expertise","score":0.3499999940395355}],"concepts":[{"id":"https://openalex.org/C9395851","wikidata":"https://www.wikidata.org/wiki/Q177929","display_name":"Stack (abstract data type)","level":2,"score":0.7653776407241821},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6589114665985107},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4339563846588135},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.42824649810791016},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.3356782793998718},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3241938352584839},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.32368654012680054},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.11966648697853088},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0613420307636261},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1016/j.jss.2025.112541","doi":"https://doi.org/10.1016/j.jss.2025.112541","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2025.112541","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-b","cwts-core","dongbi-c","fnege-4","jufo-3","ki-jl-1","norway-2","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2402.08801","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.08801","pdf_url":"https://arxiv.org/pdf/2402.08801","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:orbilu.uni.lu:10993/65324","doi":null,"is_oa":true,"landing_page_url":"https://orbilu.uni.lu/handle/10993/65324","pdf_url":"https://orbilu.uni.lu/bitstream/10993/65324/1/2402.08801v1.pdf","source":{"id":"https://openalex.org/S4306401815","display_name":"Open Repository and Bibliography (University of Luxembourg)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I186903577","host_organization_name":"University of Luxembourg","host_organization_lineage":["https://openalex.org/I186903577"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Systems and Software, 112541 (2025-07-03)","raw_type":"http://purl.org/coar/resource_type/c_6501"},{"id":"pmh:oai:publications.polymtl.ca:66641","doi":null,"is_oa":true,"landing_page_url":"https://publications.polymtl.ca/66641/","pdf_url":"https://publications.polymtl.ca/66641/1/2025_DaSilva_LLMs_Stack_Overflow_discussions_reliability.pdf","source":{"id":"https://openalex.org/S4306401013","display_name":"PolyPublie (\u00c9cole Polytechnique de Montr\u00e9al)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I45683168","host_organization_name":"Polytechnique Montr\u00e9al","host_organization_lineage":["https://openalex.org/I45683168"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Da Silva, L., Samhi, J., & Khomh, F. (2025). LLMs and Stack Overflow discussions: reliability, impact, and challenges. Journal of Systems and Software, 230, 112541 (21 pages).","raw_type":"http://purl.org/coar/resource_type/c_6501"},{"id":"doi:10.48550/arxiv.2402.08801","doi":"https://doi.org/10.48550/arxiv.2402.08801","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.08801","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1016/j.jss.2025.112541","doi":"https://doi.org/10.1016/j.jss.2025.112541","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.jss.2025.112541","pdf_url":null,"source":{"id":"https://openalex.org/S37879656","display_name":"Journal of Systems and Software","issn_l":"0164-1212","issn":["0164-1212","1873-1228"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-b","cwts-core","dongbi-c","fnege-4","jufo-3","ki-jl-1","norway-2","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Systems and Software","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309949","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95"},{"id":"https://openalex.org/F4320320994","display_name":"Canada Research Chairs","ror":"https://ror.org/0517h6h17"},{"id":"https://openalex.org/F4320331165","display_name":"Fonds de recherche du Qu\u00e9bec","ror":"https://ror.org/00w3qhf76"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4411993048.pdf","grobid_xml":"https://content.openalex.org/works/W4411993048.grobid-xml"},"referenced_works_count":68,"referenced_works":["https://openalex.org/W136732505","https://openalex.org/W1972978214","https://openalex.org/W1974311367","https://openalex.org/W1978394996","https://openalex.org/W1991613282","https://openalex.org/W2033536673","https://openalex.org/W2056894403","https://openalex.org/W2125313811","https://openalex.org/W2143354134","https://openalex.org/W2168745915","https://openalex.org/W2367359560","https://openalex.org/W2506392009","https://openalex.org/W2524168323","https://openalex.org/W2604794021","https://openalex.org/W2771796168","https://openalex.org/W2789570312","https://openalex.org/W2794992746","https://openalex.org/W2809426407","https://openalex.org/W2919976030","https://openalex.org/W2947681066","https://openalex.org/W2951474986","https://openalex.org/W2978835257","https://openalex.org/W2990737658","https://openalex.org/W2991900764","https://openalex.org/W2996747854","https://openalex.org/W3000437183","https://openalex.org/W3037862795","https://openalex.org/W3097045170","https://openalex.org/W3098605233","https://openalex.org/W3137972920","https://openalex.org/W4221159672","https://openalex.org/W4226265546","https://openalex.org/W4230081186","https://openalex.org/W4247737564","https://openalex.org/W4248364726","https://openalex.org/W4252107259","https://openalex.org/W4255973387","https://openalex.org/W4281763794","https://openalex.org/W4301393026","https://openalex.org/W4318718936","https://openalex.org/W4322631505","https://openalex.org/W4323655724","https://openalex.org/W4353007481","https://openalex.org/W4361289889","https://openalex.org/W4365512576","https://openalex.org/W4366549000","https://openalex.org/W4367059011","https://openalex.org/W4367672983","https://openalex.org/W4368250038","https://openalex.org/W4376167329","https://openalex.org/W4383268189","https://openalex.org/W4384918448","https://openalex.org/W4384920109","https://openalex.org/W4385384800","https://openalex.org/W4385500670","https://openalex.org/W4386270137","https://openalex.org/W4387007101","https://openalex.org/W4387143034","https://openalex.org/W4392563948","https://openalex.org/W4396832043","https://openalex.org/W4400582732","https://openalex.org/W4401544263","https://openalex.org/W4402462263","https://openalex.org/W4402665833","https://openalex.org/W4405796591","https://openalex.org/W4412164313","https://openalex.org/W7133238093","https://openalex.org/W7155906632"],"related_works":["https://openalex.org/W4402462263","https://openalex.org/W4387075537","https://openalex.org/W4377203175","https://openalex.org/W4414814005","https://openalex.org/W4414348197","https://openalex.org/W7055442230","https://openalex.org/W7213454280","https://openalex.org/W6884901184","https://openalex.org/W4396832043","https://openalex.org/W4416331034"],"abstract_inverted_index":{"Since":[0],"its":[1,62,65],"release":[2],"in":[3,49,113,160],"November":[4],"2022,":[5],"ChatGPT":[6,34,141],"has":[7,35,164],"shaken":[8],"up":[9],"Stack":[10,86,111,128],"Overflow,":[11],"the":[12,39,43,50,73,101,114,171,177],"premier":[13],"platform":[14],"for":[15,152,187],"developers\u2019":[16],"queries":[17],"on":[18,76],"programming":[19],"and":[20,88,106,118,132,142,179,184,193],"software":[21],"development.":[22],"Demonstrating":[23],"an":[24,80],"ability":[25],"to":[26,31,92,99,109],"generate":[27],"instant,":[28],"human-like":[29],"responses":[30],"technical":[32],"questions,":[33],"ignited":[36],"debates":[37],"within":[38],"developer":[40],"community":[41],"about":[42],"evolving":[44],"role":[45],"of":[46,52,103,173,181],"human-driven":[47],"platforms":[48],"age":[51],"generative":[53],"AI.":[54],"Two":[55],"months":[56],"after":[57],"ChatGPT\u2019s":[58],"release,":[59],"Meta":[60],"released":[61],"answer":[63],"with":[64,127],"own":[66],"Large":[67],"Language":[68],"Model":[69],"(LLM)":[70],"called":[71],"LLaMA:":[72],"race":[74],"was":[75],".":[77],"We":[78],"conducted":[79],"empirical":[81,137],"study":[82],"analyzing":[83],"questions":[84],"from":[85],"Overflow":[87,112,129],"using":[89],"these":[90],"LLMs":[91,121,134,183],"address":[93],"them.":[94],"This":[95],"way,":[96],"we":[97,168],"aim":[98],"quantify":[100],"reliability":[102],"LLMs\u2019":[104],"answers":[105],"their":[107],"potential":[108],"replace":[110],"long":[115],"term;":[116],"identify":[117],"understand":[119],"why":[120],"fail;":[122],"measure":[123],"users\u2019":[124],"activity":[125,163],"evolution":[126],"over":[130],"time;":[131],"compare":[133],"together.":[135],"Our":[136],"results":[138],"are":[139],"unequivocal:":[140],"LLaMA":[143],"challenge":[144],"human":[145],"expertise,":[146],"yet":[147],"do":[148],"not":[149],"outperform":[150],"it":[151],"some":[153],"domains":[154],",":[155],"while":[156],"a":[157],"significant":[158],"decline":[159],"user":[161],"posting":[162],"been":[165],"observed.":[166],"Furthermore,":[167],"also":[169],"discuss":[170],"impact":[172],"our":[174],"findings":[175],"regarding":[176],"usage":[178],"development":[180],"new":[182],"provide":[185],"guidelines":[186],"future":[188],"challenges":[189],"faced":[190],"by":[191],"users":[192],"researchers.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":16},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":7}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}