{"id":"https://openalex.org/W4400519732","doi":"https://doi.org/10.1145/3641525.3663629","title":"Reproscreener: Leveraging LLMs for Assessing Computational Reproducibility of Machine Learning Pipelines","display_name":"Reproscreener: Leveraging LLMs for Assessing Computational Reproducibility of Machine Learning Pipelines","publication_year":2024,"publication_date":"2024-06-18","ids":{"openalex":"https://openalex.org/W4400519732","doi":"https://doi.org/10.1145/3641525.3663629"},"language":"en","primary_location":{"id":"doi:10.1145/3641525.3663629","doi":"https://doi.org/10.1145/3641525.3663629","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3641525.3663629","pdf_url":null,"source":{"id":"https://openalex.org/S7407105039","display_name":"ACM Conference on Reproducibility and Replicability (ACM REP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4400519732"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054342522","display_name":"Adhithya Bhaskar","orcid":"https://orcid.org/0000-0003-4349-105X","observed_orcids":["https://orcid.org/0000-0003-4349-105X"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adhithya Bhaskar","raw_affiliation_strings":["University of Southern California, United States"],"raw_orcid":"https://orcid.org/0000-0003-4349-105X","affiliations":[{"raw_affiliation_string":"University of Southern California, United States","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071572173","display_name":"Victoria Stodden","orcid":"https://orcid.org/0000-0003-2015-7825","observed_orcids":["https://orcid.org/0000-0003-2015-7825"]},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Victoria Stodden","raw_affiliation_strings":["University of Southern California, United States"],"raw_orcid":"https://orcid.org/0000-0003-2015-7825","affiliations":[{"raw_affiliation_string":"University of Southern California, United States","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1174212"],"apc_list":null,"apc_paid":null,"fwci":4.7259,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.95407407,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"101","last_page":"109"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9771000146865845,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9771000146865845,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.007799999788403511,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.003599999938160181,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computational-reproducibility","display_name":"computational reproducibility","score":0.9419999718666077},{"id":"https://openalex.org/keywords/machine-learning-pipelines","display_name":"machine learning pipelines","score":0.9789999723434448},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.781000018119812},{"id":"https://openalex.org/keywords/reproducibility-assessment","display_name":"reproducibility assessment","score":0.8650000095367432},{"id":"https://openalex.org/keywords/reproducibility-metrics","display_name":"reproducibility metrics","score":0.75},{"id":"https://openalex.org/keywords/publishing-standards","display_name":"publication standards","score":0.5419999957084656},{"id":"https://openalex.org/keywords/reproducibility-verification","display_name":"reproducibility verification","score":0.6010000109672546}],"concepts":[{"id":"https://openalex.org/C175309249","wikidata":"https://www.wikidata.org/wiki/Q725864","display_name":"Pipeline transport","level":2,"score":0.7592353224754333},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.714698076248169},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6516478061676025},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.12715718150138855},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07022574543952942},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.06445875763893127},{"id":"https://openalex.org/C87717796","wikidata":"https://www.wikidata.org/wiki/Q146326","display_name":"Environmental engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3641525.3663629","doi":"https://doi.org/10.1145/3641525.3663629","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3641525.3663629","pdf_url":null,"source":{"id":"https://openalex.org/S7407105039","display_name":"ACM Conference on Reproducibility and Replicability (ACM REP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G6977025781","display_name":null,"funder_award_id":"OAC 2138773","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W2605897695","https://openalex.org/W2751343396","https://openalex.org/W2788572835","https://openalex.org/W2793484265","https://openalex.org/W2793993424","https://openalex.org/W2803067141","https://openalex.org/W2808166412","https://openalex.org/W2886900553","https://openalex.org/W2970737146","https://openalex.org/W2970771982","https://openalex.org/W3036942246","https://openalex.org/W3096956705","https://openalex.org/W3111004365","https://openalex.org/W3124069781","https://openalex.org/W3149889935","https://openalex.org/W3176376493","https://openalex.org/W3192401439","https://openalex.org/W4200553326","https://openalex.org/W4206973253","https://openalex.org/W4213127247","https://openalex.org/W4283361424","https://openalex.org/W4284974794","https://openalex.org/W4302426080","https://openalex.org/W4382362073","https://openalex.org/W4389524502"],"related_works":["https://openalex.org/W7204189871","https://openalex.org/W7218604795","https://openalex.org/W6912575991","https://openalex.org/W7163523772","https://openalex.org/W7165000826","https://openalex.org/W7169737336","https://openalex.org/W7161706840","https://openalex.org/W4389771368","https://openalex.org/W4392921853","https://openalex.org/W7160297927"],"abstract_inverted_index":{"The":[0],"increasing":[1],"reliance":[2],"on":[3,49],"machine":[4,60,115],"learning":[5,61,116],"models":[6],"in":[7,74,136],"scientific":[8],"research":[9,62],"and":[10,14,53,67,72,130,134,180,183],"day-to-day":[11],"applications":[12],"\u2013":[13,22],"the":[15,78,147,157,161,166,173,185],"near-opacity":[16],"of":[17,80,83,114,156,160,176,190],"their":[18],"associated":[19],"computational":[20],"methods":[21],"creates":[23],"a":[24,81,93,107,123,152],"widely":[25],"recognized":[26],"need":[27],"to":[28,31,47,51,63,126,131,150,172,197],"enable":[29],"others":[30],"verify":[32],"results":[33],"coming":[34],"from":[35],"Machine":[36,87,162],"Learning":[37,88,163],"Pipelines.":[38],"In":[39],"this":[40],"work":[41],"we":[42,90,145],"use":[43],"an":[44],"empirical":[45,120],"approach":[46],"build":[48],"efforts":[50],"define":[52],"deploy":[54],"structured":[55],"publication":[56,84],"standards":[57,85],"that":[58],"allow":[59],"be":[64],"automatically":[65],"assessed":[66],"verified,":[68],"enabling":[69],"greater":[70],"reliability":[71],"trust":[73],"results.":[75],"To":[76],"automate":[77],"assessment":[79,105,143,155],"set":[82],"for":[86],"Pipelines":[89],"developed":[91],"Reproscreener;":[92],"novel,":[94],"open-source":[95],"software":[96],"tool":[97],"(see":[98],"https://reproscreener.org/).":[99],"We":[100,140,168],"benchmark":[101],"Reproscreener\u2019s":[102,128],"automatic":[103,174],"reproducibility":[104,138,142,177,186],"against":[106],"novel":[108,153],"manually":[109],"labeled":[110],"\u201cgold":[111],"standard\u201d":[112],"dataset":[113],"arXiv":[117],"preprints.":[118],"Our":[119],"evaluation":[121,188],"has":[122],"dual":[124],"goal:":[125],"assess":[127],"performance;":[129],"uncover":[132],"gaps":[133],"opportunities":[135],"current":[137],"standards.":[139],"develop":[141],"metrics":[144],"called":[146,165],"Repo":[148],"Metrics":[149],"provide":[151],"overall":[154],"re-executability":[158],"potential":[159],"Pipeline,":[164],"ReproScore.":[167],"used":[169],"two":[170],"approaches":[171],"identification":[175],"metrics,":[178],"keywords":[179],"LLM":[181],"tools,":[182],"found":[184],"metric":[187],"performance":[189],"Large":[191],"Language":[192],"Model":[193],"(LLM)":[194],"tools":[195],"superior":[196],"keyword":[198],"associations.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":4}],"updated_date":"2026-11-14T11:04:58.165337","created_date":"2024-07-12T00:00:00"}