{"id":"https://openalex.org/W4387355836","doi":"https://doi.org/10.48550/arxiv.2310.01727","title":"Can GPT-4 Replicate Empirical Software Engineering Research?","display_name":"Can GPT-4 Replicate Empirical Software Engineering Research?","publication_year":2023,"publication_date":"2023-10-03","ids":{"openalex":"https://openalex.org/W4387355836","doi":"https://doi.org/10.48550/arxiv.2310.01727"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.01727","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.01727","pdf_url":"https://arxiv.org/pdf/2310.01727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.01727","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006849693","display_name":"Jenny T. Liang","orcid":"https://orcid.org/0000-0001-6722-9959"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Liang, Jenny T.","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104178670","display_name":"Carmen Badea","orcid":"https://orcid.org/0009-0005-2249-6371"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Badea, Carmen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036278905","display_name":"Christian Bird","orcid":"https://orcid.org/0000-0002-7774-0387"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bird, Christian","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026655012","display_name":"Robert DeLine","orcid":"https://orcid.org/0000-0001-8885-8367"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"DeLine, Robert","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002962395","display_name":"Denae Ford","orcid":"https://orcid.org/0000-0003-0654-4335"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ford, Denae","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069657658","display_name":"Nicole Forsgren","orcid":"https://orcid.org/0000-0003-2263-9326"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Forsgren, Nicole","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5061605447","display_name":"Thomas Zimmermann","orcid":"https://orcid.org/0000-0003-4905-1469"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zimmermann, Thomas","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5006849693"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9846000075340271,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9832000136375427,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.6710599660873413},{"id":"https://openalex.org/keywords/social-software-engineering","display_name":"Social software engineering","score":0.6116282343864441},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6018660068511963},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.5959317088127136},{"id":"https://openalex.org/keywords/software-engineering-process-group","display_name":"Software Engineering Process Group","score":0.5954199433326721},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.5433157682418823},{"id":"https://openalex.org/keywords/software-requirements","display_name":"Software requirements","score":0.5389334559440613},{"id":"https://openalex.org/keywords/personal-software-process","display_name":"Personal software process","score":0.524979293346405},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.5202106833457947},{"id":"https://openalex.org/keywords/software-peer-review","display_name":"Software peer review","score":0.422793447971344},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4129631221294403},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.19384849071502686},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07539090514183044}],"concepts":[{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6710599660873413},{"id":"https://openalex.org/C182500959","wikidata":"https://www.wikidata.org/wiki/Q7551380","display_name":"Social software engineering","level":5,"score":0.6116282343864441},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6018660068511963},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.5959317088127136},{"id":"https://openalex.org/C44416564","wikidata":"https://www.wikidata.org/wiki/Q7554199","display_name":"Software Engineering Process Group","level":5,"score":0.5954199433326721},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.5433157682418823},{"id":"https://openalex.org/C54534927","wikidata":"https://www.wikidata.org/wiki/Q4462275","display_name":"Software requirements","level":5,"score":0.5389334559440613},{"id":"https://openalex.org/C39890963","wikidata":"https://www.wikidata.org/wiki/Q1702721","display_name":"Personal software process","level":5,"score":0.524979293346405},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.5202106833457947},{"id":"https://openalex.org/C74579156","wikidata":"https://www.wikidata.org/wiki/Q7554342","display_name":"Software peer review","level":5,"score":0.422793447971344},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4129631221294403},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.19384849071502686},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07539090514183044},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.01727","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.01727","pdf_url":"https://arxiv.org/pdf/2310.01727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.01727","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.01727","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.01727","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.01727","pdf_url":"https://arxiv.org/pdf/2310.01727","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"score":0.5099999904632568,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387355836.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2581854602","https://openalex.org/W4383428648","https://openalex.org/W2139942891","https://openalex.org/W2041796194","https://openalex.org/W2052544251","https://openalex.org/W1985363770","https://openalex.org/W2120376006","https://openalex.org/W2163383818","https://openalex.org/W2368620653","https://openalex.org/W63204633"],"abstract_inverted_index":{"Empirical":[0],"software":[1,15,40,73,90,104,119,135,155,167,207,246,257,267],"engineering":[2,16,41,74,105,120,136,156,168,208,247,258],"research":[3,47,67,121,137,169,259],"on":[4,48,122,152],"production":[5,29],"systems":[6,30],"has":[7],"brought":[8],"forth":[9],"a":[10,25,63,160,179,211,229,243],"better":[11],"understanding":[12,65],"of":[13,28,36,57,66,117,181,214,231,245],"the":[14,34,185,215,221,232,235],"process":[17],"for":[18,149,253,256],"practitioners":[19,42],"and":[20,69,92,100,146,175],"researchers":[21],"alike.":[22],"However,":[23,234],"only":[24],"small":[26,239],"subset":[27,230],"is":[31,191],"studied,":[32],"limiting":[33],"impact":[35],"this":[37,52,108],"research.":[38,106],"While":[39],"could":[43,97],"benefit":[44],"from":[45,184],"replicating":[46],"their":[49,127,142],"own":[50,55],"data,":[51],"poses":[53],"its":[54],"set":[56],"challenges,":[58],"since":[59],"performing":[60],"replications":[61,116],"requires":[62],"deep":[64],"methodologies":[68],"subtle":[70],"nuances":[71],"in":[72,87,133,266],"data.":[75,124,209],"Given":[76],"that":[77,189,202,220],"large":[78],"language":[79],"models":[80,96],"(LLMs),":[81],"such":[82],"as":[83,139,141,260,262],"GPT-4,":[84],"show":[85],"promise":[86],"tackling":[88],"both":[89],"engineering-":[91],"science-related":[93],"tasks,":[94],"these":[95],"help":[98],"replicate":[99],"thus":[101],"democratize":[102],"empirical":[103,118,134,154],"In":[107,210],"paper,":[109],"we":[110,218],"examine":[111],"GPT-4's":[112],"abilities":[113],"to":[114,129,144,193,199],"perform":[115,159],"new":[123],"We":[125,158,187],"study":[126,162],"ability":[128,143],"surface":[130,194],"assumptions":[131,174],"made":[132],"methodologies,":[138],"well":[140,261],"plan":[145],"generate":[147,200],"code":[148,223,236],"analysis":[150,176,213],"pipelines":[151],"seven":[153],"papers.":[157,186],"user":[161],"with":[163,166],"14":[164],"participants":[165],"expertise,":[170],"who":[171],"evaluate":[172],"GPT-4-generated":[173,222],"plans":[177],"(i.e.,":[178],"list":[180],"module":[182],"specifications)":[183],"find":[188,219],"GPT-4":[190],"able":[192],"correct":[195,225],"assumptions,":[196],"but":[197],"struggles":[198],"ones":[201],"apply":[203],"common":[204],"knowledge":[205],"about":[206],"manual":[212],"generated":[216],"code,":[217],"contains":[224,237],"high-level":[226],"logic,":[227],"given":[228],"methodology.":[233],"many":[238],"implementation-level":[240],"errors,":[241],"reflecting":[242],"lack":[244],"knowledge.":[248],"Our":[249],"findings":[250],"have":[251],"implications":[252],"leveraging":[254],"LLMs":[255],"practitioner":[263],"data":[264],"scientists":[265],"teams.":[268]},"counts_by_year":[],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}