{"id":"https://openalex.org/W4416034576","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.539","title":"AIRepr: An Analyst-Inspector Framework for Evaluating Reproducibility of LLMs in Data Science","display_name":"AIRepr: An Analyst-Inspector Framework for Evaluating Reproducibility of LLMs in Data Science","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416034576","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.539"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.539","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.539","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.539","pdf_url":"https://aclanthology.org/2025.findings-emnlp.539.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2025.findings-emnlp.539.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Qiuhai Zeng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qiuhai Zeng","raw_affiliation_strings":["Pennsylvania State University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pennsylvania State University ,","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108988088","display_name":"Claire Jin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Claire Jin","raw_affiliation_strings":["Carnegie Mellon University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University ,","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100408878","display_name":"Xinyue Wang","orcid":"https://orcid.org/0000-0002-8177-3712","observed_orcids":["https://orcid.org/0000-0002-8177-3712","https://orcid.org/0009-0000-9985-621X"]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xinyue Wang","raw_affiliation_strings":["Pennsylvania State University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pennsylvania State University ,","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021505985","display_name":"Yuhan Zheng","orcid":"https://orcid.org/0000-0002-4136-0673","observed_orcids":["https://orcid.org/0000-0002-4136-0673"]},"institutions":[{"id":"https://openalex.org/I1310145890","display_name":"International Monetary Fund","ror":"https://ror.org/05m4rmw09","country_code":"US","type":"other","lineage":["https://openalex.org/I1310145890"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuhan Zheng","raw_affiliation_strings":["International Monetary Fund"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"International Monetary Fund","institution_ids":["https://openalex.org/I1310145890"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029682068","display_name":"Qunhua Li","orcid":"https://orcid.org/0000-0003-0675-7648","observed_orcids":["https://orcid.org/0000-0003-0675-7648"]},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qunhua Li","raw_affiliation_strings":["Pennsylvania State University ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Pennsylvania State University ,","institution_ids":["https://openalex.org/I130769515"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":20.4418,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":{"value":0.99555753,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"10170","last_page":"10201"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9229000210762024,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9229000210762024,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.04820000007748604,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.008799999952316284,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9660000205039978},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.8659999966621399},{"id":"https://openalex.org/keywords/llm-generated-code","display_name":"LLM-generated code","score":0.8190000057220459},{"id":"https://openalex.org/keywords/data-analysis-pipeline","display_name":"data analysis pipeline","score":0.906000018119812},{"id":"https://openalex.org/keywords/prompting-strategies","display_name":"prompting strategies","score":0.6340000033378601},{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.5590000152587891},{"id":"https://openalex.org/keywords/human-ai-collaboration","display_name":"human-AI collaboration","score":0.9480000138282776},{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.42800000309944153}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4675000011920929},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.42410001158714294},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.3319999873638153},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.33149999380111694},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.29499998688697815},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2773999869823456},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26409998536109924},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2572999894618988},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.2524000108242035},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.2492000013589859}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2025.findings-emnlp.539","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.539","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.539","pdf_url":"https://aclanthology.org/2025.findings-emnlp.539.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2502.16395","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.16395","pdf_url":"https://arxiv.org/pdf/2502.16395","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.16395","doi":"https://doi.org/10.48550/arxiv.2502.16395","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.16395","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.539","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.539","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.539","pdf_url":"https://aclanthology.org/2025.findings-emnlp.539.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416034576.pdf","grobid_xml":"https://content.openalex.org/works/W4416034576.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W4221161695","https://openalex.org/W4304195432","https://openalex.org/W4327810158","https://openalex.org/W4362598382","https://openalex.org/W4384812233","https://openalex.org/W4385963839","https://openalex.org/W4386395478","https://openalex.org/W4389217371","https://openalex.org/W4392012003","https://openalex.org/W4392271261","https://openalex.org/W4392492277","https://openalex.org/W4400373784","https://openalex.org/W4401306886","https://openalex.org/W4402670585","https://openalex.org/W4402671883","https://openalex.org/W4404261785","https://openalex.org/W4407760093","https://openalex.org/W4408867481","https://openalex.org/W4409885870","https://openalex.org/W4411119607","https://openalex.org/W4414281281","https://openalex.org/W4414836306","https://openalex.org/W4415797419","https://openalex.org/W6966986738"],"related_works":["https://openalex.org/W7208720693","https://openalex.org/W7129400145","https://openalex.org/W7169563478","https://openalex.org/W7162245959","https://openalex.org/W7167667599","https://openalex.org/W7163523772","https://openalex.org/W7165000826","https://openalex.org/W4416941932","https://openalex.org/W7125434856","https://openalex.org/W7164479763"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"are":[4],"increasingly":[5],"used":[6],"to":[7,30,60,74],"automate":[8],"data":[9,15,100,170],"analysis":[10,101],"through":[11],"executable":[12],"code":[13,44,68,172],"generation.Yet,":[14],"science":[16],"tasks":[17,133],"often":[18],"admit":[19],"multiple":[20],"statistically":[21],"valid":[22],"solutions,":[23],"e.g.different":[24],"modeling":[25],"strategies,":[26],"making":[27],"it":[28,50,70],"critical":[29],"understand":[31],"the":[32,62,96],"reasoning":[33],"behind":[34],"analyses,":[35,149],"not":[36],"just":[37],"their":[38],"outcomes.While":[39],"manual":[40],"review":[41],"of":[42,98],"LLM-generated":[43,78,99],"can":[45],"help":[46],"ensure":[47],"statistical":[48,107],"soundness,":[49],"is":[51,59,104,173],"laborintensive":[52],"and":[53,94,109,120,131,150,165],"requires":[54],"expertise.A":[55],"more":[56,147],"scalable":[57],"approach":[58],"evaluate":[61],"underlying":[63],"workflows-the":[64],"logical":[65],"plans":[66],"guiding":[67],"generation.However,":[69],"remains":[71],"unclear":[72],"how":[73],"assess":[75],"whether":[76],"an":[77,88],"workflow":[79],"supports":[80,110],"reproducible":[81],"implementations.To":[82],"address":[83],"this,":[84],"we":[85],"present":[86],"AIRepr,":[87],"Analyst-Inspector":[89],"framework":[90,103],"for":[91,162],"automatically":[92],"evaluating":[93],"improving":[95],"reproducibility":[97,144],"workflows.Our":[102],"grounded":[105],"in":[106,169],"principles":[108],"scalable,":[111],"automated":[112],"assessment.We":[113],"introduce":[114],"two":[115],"novel":[116],"reproducibilityenhancing":[117],"prompting":[118,125],"strategies":[119],"benchmark":[121],"them":[122],"against":[123],"standard":[124],"across":[126],"15":[127],"analyst-inspector":[128],"LLM":[129],"pairs":[130],"1,032":[132],"from":[134],"three":[135],"public":[136],"benchmarks.Our":[137],"findings":[138],"show":[139],"that":[140,151],"workflows":[141],"with":[142],"higher":[143],"also":[145],"yield":[146],"accurate":[148],"reproducibility-enhancing":[152],"prompts":[153],"substantially":[154],"improve":[155],"both":[156],"metrics.This":[157],"work":[158],"provides":[159],"a":[160],"foundation":[161],"transparent,":[163],"reliable,":[164],"efficient":[166],"human-AI":[167],"collaboration":[168],"science.Our":[171],"publicly":[174],"available":[175],"1":[176],".":[177]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":4}],"updated_date":"2026-11-12T12:28:57.059296","created_date":"2025-11-08T00:00:00"}