{"id":"https://openalex.org/W4412944691","doi":"https://doi.org/10.18653/v1/2025.findings-acl.1210","title":"REPRO-Bench: Can Agentic AI Systems Assess the Reproducibility of Social Science Research?","display_name":"REPRO-Bench: Can Agentic AI Systems Assess the Reproducibility of Social Science Research?","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412944691","doi":"https://doi.org/10.18653/v1/2025.findings-acl.1210"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.1210","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.1210","pdf_url":"https://aclanthology.org/2025.findings-acl.1210.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.1210.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060192796","display_name":"Chuxuan Hu","orcid":"https://orcid.org/0009-0001-3746-2722"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chuxuan Hu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100601568","display_name":"Liyun Zhang","orcid":"https://orcid.org/0000-0002-2394-9521"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liyun Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113370015","display_name":"Yeji Lim","orcid":"https://orcid.org/0009-0000-4485-6597"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yeji Lim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Aum Wadhwani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aum Wadhwani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048021375","display_name":"Austin Peters","orcid":"https://orcid.org/0000-0003-0200-1042"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Austin Peters","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5072348548","display_name":"Daniel Kang","orcid":"https://orcid.org/0000-0001-9860-9938"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Daniel Kang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.6716,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.8715415,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"23616","last_page":"23626"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.8773000240325928,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.8773000240325928,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.7372800707817078},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5326153039932251},{"id":"https://openalex.org/keywords/bench-to-bedside","display_name":"Bench to bedside","score":0.4750808775424957},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4073525071144104},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3327797055244446},{"id":"https://openalex.org/keywords/medical-physics","display_name":"Medical physics","score":0.10733094811439514},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.09911060333251953},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08566313982009888},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07453379034996033}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.7372800707817078},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5326153039932251},{"id":"https://openalex.org/C3020000666","wikidata":"https://www.wikidata.org/wiki/Q17112037","display_name":"Bench to bedside","level":2,"score":0.4750808775424957},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4073525071144104},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3327797055244446},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.10733094811439514},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.09911060333251953},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08566313982009888},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07453379034996033}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.1210","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.1210","pdf_url":"https://aclanthology.org/2025.findings-acl.1210.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.1210","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.1210","pdf_url":"https://aclanthology.org/2025.findings-acl.1210.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Gender equality","score":0.5600000023841858,"id":"https://metadata.un.org/sdg/5"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412944691.pdf","grobid_xml":"https://content.openalex.org/works/W4412944691.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W4324138256"],"abstract_inverted_index":{"Assessing":[0],"the":[1,61,107,110,114,119,128,148,166],"reproducibility":[2,108,129,187],"of":[3,86,109,130,154],"social":[4,93,131],"science":[5,94,132],"papers":[6,44,133],"is":[7,18],"essential":[8],"for":[9,41],"promoting":[10],"rigor":[11],"in":[12,22,72],"research":[13,43],"processes,":[14],"but":[15],"manual":[16],"assessment":[17],"costly.With":[19],"recent":[20],"advances":[21],"agentic":[23],"AI":[24,27,143,179],"systems":[25],"(i.e.,":[26],"agents),":[28],"we":[29,81,161],"seek":[30],"to":[31,35,137,184],"evaluate":[32,140],"their":[33,58],"capability":[34],"automate":[36,185],"this":[37],"process.However,":[38],"existing":[39,171],"benchmarks":[40],"reproducing":[42,49],"(1)":[45],"focus":[46],"solely":[47],"on":[48,113,127,145,157],"results":[50],"using":[51],"provided":[52],"code":[53],"and":[54,67,75,118],"data":[55,73],"without":[56],"assessing":[57,106],"consistency":[59],"with":[60,96,105,134,147],"paper,":[62],"(2)":[63],"oversimplify":[64],"real-world":[65,138,186],"scenarios,":[66],"(3)":[68],"lack":[69],"necessary":[70],"diversity":[71],"formats":[74],"programming":[76],"languages.To":[77],"address":[78],"these":[79],"issues,":[80],"introduce":[82],"REPRO-BENCH,":[83,146],"a":[84,92,97],"collection":[85],"112":[87],"task":[88],"instances,":[89],"each":[90],"representing":[91],"paper":[95,111,116],"publicly":[98],"available":[99],"reproduction":[100,121],"report.The":[101],"agents":[102,144,172,180],"are":[103],"tasked":[104],"based":[112],"original":[115],"PDF":[117],"corresponding":[120],"package.REPRO-BENCH":[122],"features":[123],"end-to-end":[124],"evaluation":[125],"tasks":[126],"complexity":[135],"comparable":[136],"assessments.We":[139],"three":[141],"representative":[142],"best-performing":[149],"agent":[150],"achieving":[151],"an":[152],"accuracy":[153,168],"only":[155],"21.4%.Building":[156],"our":[158],"empirical":[159],"analysis,":[160],"develop":[162],"REPRO-AGENT,":[163],"which":[164],"improves":[165],"highest":[167],"achieved":[169],"by":[170,173],"71%.We":[174],"conclude":[175],"that":[176],"more":[177],"advanced":[178],"should":[181],"be":[182],"developed":[183],"assessment.":[188]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}