{"id":"https://openalex.org/W4400519732","doi":"https://doi.org/10.1145/3641525.3663629","title":"Reproscreener: Leveraging LLMs for Assessing Computational Reproducibility of Machine Learning Pipelines","display_name":"Reproscreener: Leveraging LLMs for Assessing Computational Reproducibility of Machine Learning Pipelines","publication_year":2024,"publication_date":"2024-06-18","ids":{"openalex":"https://openalex.org/W4400519732","doi":"https://doi.org/10.1145/3641525.3663629"},"language":"en","primary_location":{"id":"doi:10.1145/3641525.3663629","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3641525.3663629","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054342522","display_name":"Adhithya Bhaskar","orcid":"https://orcid.org/0000-0003-4349-105X"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I2800817003","display_name":"California Southern University","ror":"https://ror.org/058zz0t50","country_code":"US","type":"education","lineage":["https://openalex.org/I2800817003"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adhithya Bhaskar","raw_affiliation_strings":["University of Southern California, United States"],"raw_orcid":"https://orcid.org/0000-0003-4349-105X","affiliations":[{"raw_affiliation_string":"University of Southern California, United States","institution_ids":["https://openalex.org/I1174212","https://openalex.org/I2800817003"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071572173","display_name":"Victoria Stodden","orcid":"https://orcid.org/0000-0003-2015-7825"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I2800817003","display_name":"California Southern University","ror":"https://ror.org/058zz0t50","country_code":"US","type":"education","lineage":["https://openalex.org/I2800817003"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Victoria Stodden","raw_affiliation_strings":["University of Southern California, United States"],"raw_orcid":"https://orcid.org/0000-0003-2015-7825","affiliations":[{"raw_affiliation_string":"University of Southern California, United States","institution_ids":["https://openalex.org/I1174212","https://openalex.org/I2800817003"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.0668,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.7739651,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"101","last_page":"109"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9370999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pipeline-transport","display_name":"Pipeline transport","score":0.7592353224754333},{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.714698076248169},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6516478061676025},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.12715718150138855},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07022574543952942},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.06445875763893127}],"concepts":[{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.7592353224754333},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.714698076248169},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6516478061676025},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.12715718150138855},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07022574543952942},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.06445875763893127},{"id":"https://openalex.org/C87717796","wikidata":"https://www.wikidata.org/wiki/Q146326","display_name":"Environmental engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3641525.3663629","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3641525.3663629","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6977025781","display_name":null,"funder_award_id":"OAC 2138773","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W2605897695","https://openalex.org/W2751343396","https://openalex.org/W2788572835","https://openalex.org/W2793484265","https://openalex.org/W2793993424","https://openalex.org/W2803067141","https://openalex.org/W2808166412","https://openalex.org/W2886900553","https://openalex.org/W2970737146","https://openalex.org/W2970771982","https://openalex.org/W3096956705","https://openalex.org/W3111004365","https://openalex.org/W3124069781","https://openalex.org/W3149889935","https://openalex.org/W3184756548","https://openalex.org/W4200553326","https://openalex.org/W4206973253","https://openalex.org/W4213127247","https://openalex.org/W4283361424","https://openalex.org/W4284974794","https://openalex.org/W4302426080","https://openalex.org/W4382362073","https://openalex.org/W4389524502"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W3134060498","https://openalex.org/W2003643616"],"abstract_inverted_index":{"The":[0],"increasing":[1],"reliance":[2],"on":[3,49],"machine":[4,60,115],"learning":[5,61,116],"models":[6],"in":[7,74,136],"scientific":[8],"research":[9,62],"and":[10,14,53,67,72,130,134,180,183],"day-to-day":[11],"applications":[12],"\u2013":[13,22],"the":[15,78,147,157,161,166,173,185],"near-opacity":[16],"of":[17,80,83,114,156,160,176,190],"their":[18],"associated":[19],"computational":[20],"methods":[21],"creates":[23],"a":[24,81,93,107,123,152],"widely":[25],"recognized":[26],"need":[27],"to":[28,31,47,51,63,126,131,150,172,197],"enable":[29],"others":[30],"verify":[32],"results":[33],"coming":[34],"from":[35],"Machine":[36,87,162],"Learning":[37,88,163],"Pipelines.":[38],"In":[39],"this":[40],"work":[41],"we":[42,90,145],"use":[43],"an":[44],"empirical":[45,120],"approach":[46],"build":[48],"efforts":[50],"define":[52],"deploy":[54],"structured":[55],"publication":[56,84],"standards":[57,85],"that":[58],"allow":[59],"be":[64],"automatically":[65],"assessed":[66],"verified,":[68],"enabling":[69],"greater":[70],"reliability":[71],"trust":[73],"results.":[75],"To":[76],"automate":[77],"assessment":[79,105,143,155],"set":[82],"for":[86],"Pipelines":[89],"developed":[91],"Reproscreener;":[92],"novel,":[94],"open-source":[95],"software":[96],"tool":[97],"(see":[98],"https://reproscreener.org/).":[99],"We":[100,140,168],"benchmark":[101],"Reproscreener\u2019s":[102,128],"automatic":[103,174],"reproducibility":[104,138,142,177,186],"against":[106],"novel":[108,153],"manually":[109],"labeled":[110],"\u201cgold":[111],"standard\u201d":[112],"dataset":[113],"arXiv":[117],"preprints.":[118],"Our":[119],"evaluation":[121,188],"has":[122],"dual":[124],"goal:":[125],"assess":[127],"performance;":[129],"uncover":[132],"gaps":[133],"opportunities":[135],"current":[137],"standards.":[139],"develop":[141],"metrics":[144],"called":[146,165],"Repo":[148],"Metrics":[149],"provide":[151],"overall":[154],"re-executability":[158],"potential":[159],"Pipeline,":[164],"ReproScore.":[167],"used":[169],"two":[170],"approaches":[171],"identification":[175],"metrics,":[178],"keywords":[179],"LLM":[181],"tools,":[182],"found":[184],"metric":[187],"performance":[189],"Large":[191],"Language":[192],"Model":[193],"(LLM)":[194],"tools":[195],"superior":[196],"keyword":[198],"associations.":[199]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}