{"id":"https://openalex.org/W7133298801","doi":"https://doi.org/10.48550/arxiv.2603.00058","title":"PaperRepro: Automated Computational Reproducibility Assessment for Social Science Papers","display_name":"PaperRepro: Automated Computational Reproducibility Assessment for Social Science Papers","publication_year":2026,"publication_date":"2026-02-10","ids":{"openalex":"https://openalex.org/W7133298801","doi":"https://doi.org/10.48550/arxiv.2603.00058"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.00058","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.00058","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.00058","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5127993155","display_name":"Linhao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Zhang, Linhao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127896943","display_name":"Tong Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Tong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128020200","display_name":"Jinghua Piao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Piao, Jinghua","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Cui, Lizhen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cui, Lizhen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5127873298","display_name":"Yong Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yong","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5127993155"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6173999905586243,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.6173999905586243,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.08860000222921371,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.051899999380111694,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.7631999850273132},{"id":"https://openalex.org/keywords/credibility","display_name":"Credibility","score":0.6625999808311462},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6552000045776367},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6194999814033508},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4902999997138977},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.4162999987602234}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7694000005722046},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.7631999850273132},{"id":"https://openalex.org/C2780224610","wikidata":"https://www.wikidata.org/wiki/Q1530061","display_name":"Credibility","level":2,"score":0.6625999808311462},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6552000045776367},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6194999814033508},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4902999997138977},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4578999876976013},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4162999987602234},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.38370001316070557},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3402999937534332},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.31299999356269836},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.30790001153945923},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2946999967098236},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.00058","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.00058","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.00058","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.00058","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"display_name":"Gender equality","id":"https://metadata.un.org/sdg/5","score":0.4852760434150696}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Computational":[0],"reproducibility":[1,22,56,117,161,203],"is":[2,24,30],"essential":[3],"for":[4,54,154,197],"the":[5,12,34,38,92,97,102,112,144,166,179,186],"credibility":[6],"of":[7,201],"scientific":[8],"findings,":[9],"particularly":[10],"in":[11,46,175],"social":[13,159],"sciences,":[14],"where":[15],"findings":[16,36],"often":[17,61],"inform":[18],"real-world":[19],"decisions.":[20],"Manual":[21],"assessment":[23,162,204],"costly":[25],"and":[26,42,71,100,127,133,138,188,208],"time-consuming,":[27],"as":[28,108],"it":[29],"nontrivial":[31],"to":[32,64,104,125,148],"reproduce":[33],"reported":[35],"using":[37,118],"authors'":[39],"released":[40],"code":[41,103,207],"data.":[43],"Recent":[44],"advances":[45],"large":[47],"models":[48],"(LMs)":[49],"have":[50],"inspired":[51],"agent-based":[52],"approaches":[53,60],"automated":[55,202],"assessment.":[57],"However,":[58],"existing":[59],"struggle":[62],"due":[63],"limited":[65],"context":[66,137],"capacity,":[67],"inadequate":[68],"task-specific":[69,131],"tooling,":[70],"insufficient":[72],"result":[73,152],"capture.":[74],"To":[75],"address":[76],"these,":[77],"we":[78],"propose":[79],"PaperRepro,":[80],"a":[81,158,171,191],"novel":[82],"two-stage,":[83],"multi-agent":[84],"approach":[85],"that":[86],"separates":[87],"execution":[88,93,195],"from":[89],"evaluation.":[90,155],"In":[91,111],"stage,":[94,114],"agents":[95,115,126],"execute":[96],"reproduction":[98],"package":[99],"edit":[101],"capture":[105,153],"reproduced":[106],"results":[107],"explicit":[109,119],"artifacts.":[110],"evaluation":[113,200],"evaluate":[116],"evidence.":[120],"PaperRepro":[121,164],"assigns":[122],"distinct":[123],"responsibilities":[124],"equips":[128],"them":[129],"with":[130,170],"tools":[132],"expert":[134],"prompts,":[135],"mitigating":[136],"tooling":[139],"limitations.":[140],"It":[141],"further":[142,184],"maximizes":[143],"LM's":[145],"coding":[146],"capability":[147],"enable":[149],"more":[150,198],"complete":[151],"On":[156],"REPRO-Bench,":[157],"science":[160],"benchmark,":[163],"achieves":[165],"best":[167],"overall":[168],"performance,":[169],"21.9%":[172],"relative":[173],"improvement":[174],"score-agreement":[176],"accuracy":[177],"over":[178],"strongest":[180],"prior":[181],"baseline.":[182],"We":[183],"refine":[185],"benchmark":[187,192],"introduce":[189],"REPRO-Bench-S,":[190],"stratified":[193],"by":[194],"difficulty":[196],"diagnostic":[199],"systems.":[205],"Our":[206],"data":[209],"are":[210],"publicly":[211],"available":[212]},"counts_by_year":[],"updated_date":"2026-03-25T23:56:10.502304","created_date":"2026-03-04T00:00:00"}