{"id":"https://openalex.org/W4400582732","doi":"https://doi.org/10.1145/3660767","title":"Can GPT-4 Replicate Empirical Software Engineering Research?","display_name":"Can GPT-4 Replicate Empirical Software Engineering Research?","publication_year":2024,"publication_date":"2024-07-12","ids":{"openalex":"https://openalex.org/W4400582732","doi":"https://doi.org/10.1145/3660767"},"language":"en","primary_location":{"id":"doi:10.1145/3660767","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3660767","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3660767","source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3660767","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006849693","display_name":"Jenny T. Liang","orcid":"https://orcid.org/0000-0001-6722-9959"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jenny T. Liang","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0000-0001-6722-9959","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104178670","display_name":"Carmen Badea","orcid":"https://orcid.org/0009-0005-2249-6371"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carmen Badea","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"raw_orcid":"https://orcid.org/0009-0005-2249-6371","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036278905","display_name":"Christian Bird","orcid":"https://orcid.org/0000-0002-7774-0387"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christian Bird","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"raw_orcid":"https://orcid.org/0000-0002-7774-0387","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026655012","display_name":"Robert DeLine","orcid":"https://orcid.org/0000-0001-8885-8367"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert DeLine","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"raw_orcid":"https://orcid.org/0000-0001-8885-8367","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002962395","display_name":"Denae Ford","orcid":"https://orcid.org/0000-0003-0654-4335"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Denae Ford","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"raw_orcid":"https://orcid.org/0000-0003-0654-4335","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069657658","display_name":"Nicole Forsgren","orcid":"https://orcid.org/0000-0003-2263-9326"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicole Forsgren","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"raw_orcid":"https://orcid.org/0000-0003-2263-9326","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061605447","display_name":"Thomas Zimmermann","orcid":"https://orcid.org/0000-0003-4905-1469"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thomas Zimmermann","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"raw_orcid":"https://orcid.org/0000-0003-4905-1469","affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5006849693"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":11.1636,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.98301395,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"1","issue":"FSE","first_page":"1330","last_page":"1353"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.6702356934547424},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6321597695350647},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.6146384477615356},{"id":"https://openalex.org/keywords/social-software-engineering","display_name":"Social software engineering","score":0.6041624546051025},{"id":"https://openalex.org/keywords/software-engineering-process-group","display_name":"Software Engineering Process Group","score":0.5711467266082764},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.568185567855835},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.5226811766624451},{"id":"https://openalex.org/keywords/software-requirements","display_name":"Software requirements","score":0.5194244384765625},{"id":"https://openalex.org/keywords/personal-software-process","display_name":"Personal software process","score":0.5178109407424927},{"id":"https://openalex.org/keywords/replicate","display_name":"Replicate","score":0.4363930821418762},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4273624122142792},{"id":"https://openalex.org/keywords/software-peer-review","display_name":"Software peer review","score":0.41658106446266174},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.32111334800720215},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.20882782340049744},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07781225442886353}],"concepts":[{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6702356934547424},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6321597695350647},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.6146384477615356},{"id":"https://openalex.org/C182500959","wikidata":"https://www.wikidata.org/wiki/Q7551380","display_name":"Social software engineering","level":5,"score":0.6041624546051025},{"id":"https://openalex.org/C44416564","wikidata":"https://www.wikidata.org/wiki/Q7554199","display_name":"Software Engineering Process Group","level":5,"score":0.5711467266082764},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.568185567855835},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.5226811766624451},{"id":"https://openalex.org/C54534927","wikidata":"https://www.wikidata.org/wiki/Q4462275","display_name":"Software requirements","level":5,"score":0.5194244384765625},{"id":"https://openalex.org/C39890963","wikidata":"https://www.wikidata.org/wiki/Q1702721","display_name":"Personal software process","level":5,"score":0.5178109407424927},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.4363930821418762},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4273624122142792},{"id":"https://openalex.org/C74579156","wikidata":"https://www.wikidata.org/wiki/Q7554342","display_name":"Software peer review","level":5,"score":0.41658106446266174},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32111334800720215},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.20882782340049744},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07781225442886353},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3660767","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3660767","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3660767","source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3660767","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3660767","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3660767","source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.5400000214576721,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400582732.pdf"},"referenced_works_count":56,"referenced_works":["https://openalex.org/W171406816","https://openalex.org/W1984769753","https://openalex.org/W1994931507","https://openalex.org/W2011044158","https://openalex.org/W2043223258","https://openalex.org/W2046434103","https://openalex.org/W2084429940","https://openalex.org/W2101720091","https://openalex.org/W2110229593","https://openalex.org/W2146649871","https://openalex.org/W2153150125","https://openalex.org/W2163004671","https://openalex.org/W2364861723","https://openalex.org/W2757656223","https://openalex.org/W2784876765","https://openalex.org/W2883188794","https://openalex.org/W2939684257","https://openalex.org/W2964232059","https://openalex.org/W3044177754","https://openalex.org/W3081249378","https://openalex.org/W3091852855","https://openalex.org/W3091973425","https://openalex.org/W3102186774","https://openalex.org/W3104849124","https://openalex.org/W3104989070","https://openalex.org/W3123732438","https://openalex.org/W3130767973","https://openalex.org/W3151195892","https://openalex.org/W3160832179","https://openalex.org/W3173637057","https://openalex.org/W4220685986","https://openalex.org/W4220877452","https://openalex.org/W4221143046","https://openalex.org/W4225085525","https://openalex.org/W4239675444","https://openalex.org/W4280549083","https://openalex.org/W4281763794","https://openalex.org/W4284709654","https://openalex.org/W4285490371","https://openalex.org/W4292779060","https://openalex.org/W4292967768","https://openalex.org/W4300780349","https://openalex.org/W4308643318","https://openalex.org/W4360978668","https://openalex.org/W4362659486","https://openalex.org/W4366549000","https://openalex.org/W4372334257","https://openalex.org/W4384345688","https://openalex.org/W4391558520","https://openalex.org/W4391876619","https://openalex.org/W4396832182","https://openalex.org/W4396833115","https://openalex.org/W4402665833","https://openalex.org/W4402670775","https://openalex.org/W6778883912","https://openalex.org/W6902135807"],"related_works":["https://openalex.org/W2581854602","https://openalex.org/W4383428648","https://openalex.org/W2139942891","https://openalex.org/W2041796194","https://openalex.org/W2052544251","https://openalex.org/W1985363770","https://openalex.org/W2120376006","https://openalex.org/W2163383818","https://openalex.org/W2368620653","https://openalex.org/W63204633"],"abstract_inverted_index":{"Empirical":[0],"software":[1,15,40,73,90,104,119,136,156,168,208,247,258,268],"engineering":[2,16,41,74,105,120,137,157,169,209,248,259],"research":[3,47,67,121,138,170,260],"on":[4,48,122,153],"production":[5,29],"systems":[6,30],"has":[7],"brought":[8],"forth":[9],"a":[10,25,63,161,180,212,230,244],"better":[11],"understanding":[12,65],"of":[13,28,36,57,66,117,182,215,232,246],"the":[14,34,186,216,222,233,236],"process":[17],"for":[18,150,254,257],"practitioners":[19,42],"and":[20,69,92,100,147,176],"researchers":[21],"alike.":[22],"However,":[23,235],"only":[24],"small":[26,240],"subset":[27,231],"is":[31,192],"studied,":[32],"limiting":[33],"impact":[35],"this":[37,52,108],"research.":[38,106],"While":[39],"could":[43,97],"benefit":[44],"from":[45,185],"replicating":[46],"their":[49,128,143],"own":[50,55],"data,":[51],"poses":[53],"its":[54],"set":[56],"challenges,":[58],"since":[59],"performing":[60],"replications":[61,116],"requires":[62],"deep":[64],"methodologies":[68],"subtle":[70],"nuances":[71],"in":[72,87,134,267],"data.":[75,124,210],"Given":[76],"that":[77,190,203,221],"large":[78],"language":[79],"models":[80,96],"(LLMs),":[81],"such":[82],"as":[83,140,142,261,263],"GPT-4,":[84],"show":[85],"promise":[86],"tackling":[88],"both":[89],"engineering-":[91],"science-related":[93],"tasks,":[94],"these":[95],"help":[98],"replicate":[99],"thus":[101],"democratize":[102],"empirical":[103,118,135,155],"In":[107,211],"paper,":[109],"we":[110,219],"examine":[111],"GPT-4\u2019s":[112],"abilities":[113],"to":[114,130,145,194,200],"perform":[115,160],"new":[123],"We":[125,159,188],"specifically":[126],"study":[127,163],"ability":[129,144],"surface":[131,195],"assumptions":[132,175],"made":[133],"methodologies,":[139],"well":[141,262],"plan":[146],"generate":[148,201],"code":[149,224,237],"analysis":[151,177,214],"pipelines":[152],"seven":[154],"papers.":[158,187],"user":[162],"with":[164,167],"14":[165],"participants":[166],"expertise,":[171],"who":[172],"evaluate":[173],"GPT-4-generated":[174,223],"plans":[178],"(i.e.,":[179],"list":[181],"module":[183],"specifications)":[184],"find":[189,220],"GPT-4":[191],"able":[193],"correct":[196,226],"assumptions,":[197],"but":[198],"struggles":[199],"ones":[202],"apply":[204],"common":[205],"knowledge":[206],"about":[207],"manual":[213],"generated":[217],"code,":[218],"contains":[225,238],"high-level":[227],"logic,":[228],"given":[229],"methodology.":[234],"many":[239],"implementation-level":[241],"errors,":[242],"reflecting":[243],"lack":[245],"knowledge.":[249],"Our":[250],"findings":[251],"have":[252],"implications":[253],"leveraging":[255],"LLMs":[256],"practitioner":[264],"data":[265],"scientists":[266],"teams.":[269]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-04T09:04:59.091469","created_date":"2025-10-10T00:00:00"}