{"id":"https://openalex.org/W7128475124","doi":"https://doi.org/10.48550/arxiv.2602.08561","title":"Automating Computational Reproducibility in Social Science: Comparing Prompt-Based and Agent-Based Approaches","display_name":"Automating Computational Reproducibility in Social Science: Comparing Prompt-Based and Agent-Based Approaches","publication_year":2026,"publication_date":"2026-02-09","ids":{"openalex":"https://openalex.org/W7128475124","doi":"https://doi.org/10.48550/arxiv.2602.08561"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.08561","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101610987","display_name":"Syed Waqar Shah","orcid":"https://orcid.org/0000-0003-0145-5253"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Shah, Syed Mehtab Hussain","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080524660","display_name":"Frank Hopfgartner","orcid":"https://orcid.org/0000-0003-0380-6088"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hopfgartner, Frank","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5125565324","display_name":"Arnim Bleier","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bleier, Arnim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5101610987"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.906000018119812,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.906000018119812,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.01640000008046627,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.010599999688565731,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.8127999901771545},{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.7039999961853027},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5343000292778015},{"id":"https://openalex.org/keywords/missing-data","display_name":"Missing data","score":0.42750000953674316},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.4036000072956085},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3880000114440918},{"id":"https://openalex.org/keywords/cyclomatic-complexity","display_name":"Cyclomatic complexity","score":0.3736000061035156}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8127999901771545},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7835999727249146},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.7039999961853027},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5343000292778015},{"id":"https://openalex.org/C9357733","wikidata":"https://www.wikidata.org/wiki/Q6878417","display_name":"Missing data","level":2,"score":0.42750000953674316},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.4036000072956085},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3880000114440918},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3831000030040741},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.37439998984336853},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3736000061035156},{"id":"https://openalex.org/C187303228","wikidata":"https://www.wikidata.org/wiki/Q867330","display_name":"Cyclomatic complexity","level":3,"score":0.3736000061035156},{"id":"https://openalex.org/C66153294","wikidata":"https://www.wikidata.org/wiki/Q899291","display_name":"CASP","level":4,"score":0.37290000915527344},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36660000681877136},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.3465999960899353},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3230000138282776},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3034000098705292},{"id":"https://openalex.org/C103088060","wikidata":"https://www.wikidata.org/wiki/Q1062839","display_name":"Error detection and correction","level":2,"score":0.29490000009536743},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2782999873161316},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.25999999046325684},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.08561","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.08561","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.08561","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.08561","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"publisher-specific-oa","license_id":"https://openalex.org/licenses/publisher-specific-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reproducing":[0],"computational":[1,60],"research":[2],"is":[3,110],"often":[4],"assumed":[5],"to":[6,33,63,92],"be":[7],"as":[8,10],"simple":[9,90],"rerunning":[11],"the":[12,52,123],"original":[13],"code":[14],"with":[15,116,146,168],"provided":[16],"data.":[17],"In":[18],"practice,":[19],"missing":[20,94],"packages,":[21],"fragile":[22],"file":[23],"paths,":[24],"version":[25],"conflicts,":[26],"or":[27],"incomplete":[28],"logic":[29],"frequently":[30],"cause":[31],"analyses":[32,135],"fail,":[34],"even":[35],"when":[36],"materials":[37],"are":[38],"shared.":[39],"This":[40],"study":[41],"investigates":[42],"whether":[43],"large":[44],"language":[45,114],"models":[46,115],"and":[47,54,65,96,133,153,192,217],"AI":[48],"agents":[49],"can":[50,187],"automate":[51],"diagnosis":[53],"repair":[55,99,207],"of":[56,119,171,215],"such":[57],"failures,":[58],"making":[59],"results":[61,179],"easier":[62],"reproduce":[64],"verify.":[66],"We":[67],"evaluate":[68],"this":[69],"using":[70],"a":[71],"controlled":[72,209],"reproducibility":[73],"testbed":[74,204],"built":[75],"from":[76,89,143,160],"five":[77],"fully":[78],"reproducible":[79],"R-based":[80],"social":[81],"science":[82],"studies.":[83],"Realistic":[84],"failures":[85],"were":[86,101],"injected,":[87],"ranging":[88],"issues":[91],"complex":[93],"logic,":[95],"two":[97],"automated":[98,182],"workflows":[100,164],"tested":[102],"in":[103],"clean":[104],"Docker":[105],"environments.":[106],"The":[107],"first":[108],"workflow":[109],"prompt-based,":[111],"repeatedly":[112],"querying":[113],"structured":[117],"prompts":[118],"varying":[120],"context,":[121],"while":[122],"second":[124],"uses":[125],"agent-based":[126,185,218],"systems":[127],"that":[128,181],"inspect":[129],"files,":[130],"modify":[131],"code,":[132],"rerun":[134],"autonomously.":[136],"Across":[137],"prompt-based":[138,216],"runs,":[139],"reproduction":[140,194],"success":[141,169,195],"ranged":[142],"31-79":[144],"percent,":[145],"performance":[147],"strongly":[148],"influenced":[149],"by":[150],"prompt":[151],"context":[152],"error":[154,198],"complexity.":[155],"Complex":[156],"cases":[157],"benefited":[158],"most":[159],"additional":[161],"context.":[162],"Agent-based":[163],"performed":[165],"substantially":[166],"better,":[167],"rates":[170],"69-96":[172],"percent":[173],"across":[174,196],"all":[175],"complexity":[176],"levels.":[177],"These":[178],"suggest":[180],"workflows,":[183],"especially":[184],"systems,":[186],"significantly":[188],"reduce":[189],"manual":[190],"effort":[191],"improve":[193],"diverse":[197],"types.":[199],"Unlike":[200],"prior":[201],"benchmarks,":[202],"our":[203],"isolates":[205],"post-publication":[206],"under":[208],"failure":[210],"modes,":[211],"allowing":[212],"direct":[213],"comparison":[214],"approaches.":[219]},"counts_by_year":[],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2026-02-11T00:00:00"}