{"id":"https://openalex.org/W4388856981","doi":"https://doi.org/10.1145/3576915.3623130","title":"\"Get in Researchers; We're Measuring Reproducibility\": A Reproducibility Study of Machine Learning Papers in Tier 1 Security Conferences","display_name":"\"Get in Researchers; We're Measuring Reproducibility\": A Reproducibility Study of Machine Learning Papers in Tier 1 Security Conferences","publication_year":2023,"publication_date":"2023-11-15","ids":{"openalex":"https://openalex.org/W4388856981","doi":"https://doi.org/10.1145/3576915.3623130"},"language":"en","primary_location":{"id":"doi:10.1145/3576915.3623130","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3576915.3623130","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064538510","display_name":"Daniel Olszewski","orcid":"https://orcid.org/0009-0009-7807-7941"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Daniel Olszewski","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0009-0009-7807-7941","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103083230","display_name":"A. Lu","orcid":"https://orcid.org/0009-0000-2525-4478"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Allison Lu","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0009-0000-2525-4478","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093307570","display_name":"Carson Stillman","orcid":"https://orcid.org/0009-0001-4717-6316"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carson Stillman","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0009-0001-4717-6316","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106404842","display_name":"Kevin Warren","orcid":"https://orcid.org/0009-0008-9806-5887"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Warren","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0009-0008-9806-5887","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093307571","display_name":"Cole Kitroser","orcid":"https://orcid.org/0009-0000-7595-4288"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cole Kitroser","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0009-0000-7595-4288","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103219752","display_name":"Alejandro Pascual","orcid":"https://orcid.org/0009-0006-3526-9247"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alejandro Pascual","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0009-0006-3526-9247","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093307572","display_name":"Divyajyoti Ukirde","orcid":"https://orcid.org/0009-0002-4064-7119"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Divyajyoti Ukirde","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0009-0002-4064-7119","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039485542","display_name":"Kevin Butler","orcid":"https://orcid.org/0000-0002-7498-4239"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Butler","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0000-0002-7498-4239","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048941759","display_name":"Patrick Traynor","orcid":"https://orcid.org/0000-0002-7143-5189"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Patrick Traynor","raw_affiliation_strings":["University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0000-0002-7143-5189","affiliations":[{"raw_affiliation_string":"University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5064538510"],"corresponding_institution_ids":["https://openalex.org/I33213144"],"apc_list":null,"apc_paid":null,"fwci":10.7852,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.97941109,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"3433","last_page":"3459"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.956051230430603},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7478258609771729},{"id":"https://openalex.org/keywords/artifact","display_name":"Artifact (error)","score":0.5694664120674133},{"id":"https://openalex.org/keywords/academic-community","display_name":"Academic community","score":0.410366028547287},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3852057158946991},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.30623623728752136},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.12248721718788147},{"id":"https://openalex.org/keywords/library-science","display_name":"Library science","score":0.08067050576210022},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0667908787727356}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.956051230430603},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7478258609771729},{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.5694664120674133},{"id":"https://openalex.org/C2984558057","wikidata":"https://www.wikidata.org/wiki/Q162633","display_name":"Academic community","level":2,"score":0.410366028547287},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3852057158946991},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30623623728752136},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.12248721718788147},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.08067050576210022},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0667908787727356}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3576915.3623130","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3576915.3623130","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":391,"referenced_works":["https://openalex.org/W1488996941","https://openalex.org/W1501932514","https://openalex.org/W1563577331","https://openalex.org/W1575893985","https://openalex.org/W1608459536","https://openalex.org/W1714926069","https://openalex.org/W1969840808","https://openalex.org/W1971991172","https://openalex.org/W1974837338","https://openalex.org/W1989000482","https://openalex.org/W1993260518","https://openalex.org/W1993370323","https://openalex.org/W1994865005","https://openalex.org/W2006022520","https://openalex.org/W2007488200","https://openalex.org/W2011593588","https://openalex.org/W2019578814","https://openalex.org/W2025440088","https://openalex.org/W2038296020","https://openalex.org/W2044798763","https://openalex.org/W2051267297","https://openalex.org/W2052565897","https://openalex.org/W2053637704","https://openalex.org/W2053814546","https://openalex.org/W2064856281","https://openalex.org/W2066588645","https://openalex.org/W2069219260","https://openalex.org/W2069268700","https://openalex.org/W2088250206","https://openalex.org/W2092035184","https://openalex.org/W2095519872","https://openalex.org/W2108436124","https://openalex.org/W2117202485","https://openalex.org/W2120702739","https://openalex.org/W2125490153","https://openalex.org/W2139210474","https://openalex.org/W2155743899","https://openalex.org/W2205815960","https://openalex.org/W2272516773","https://openalex.org/W2337093093","https://openalex.org/W2396652156","https://openalex.org/W2424402128","https://openalex.org/W2441135008","https://openalex.org/W2469474955","https://openalex.org/W2473418344","https://openalex.org/W2508271471","https://openalex.org/W2510508396","https://openalex.org/W2525134402","https://openalex.org/W2532830080","https://openalex.org/W2532962075","https://openalex.org/W2535873859","https://openalex.org/W2536166232","https://openalex.org/W2560252021","https://openalex.org/W2561942675","https://openalex.org/W2573650634","https://openalex.org/W2575541032","https://openalex.org/W2577142429","https://openalex.org/W2578943040","https://openalex.org/W2591882872","https://openalex.org/W2612622960","https://openalex.org/W2618043096","https://openalex.org/W2628359750","https://openalex.org/W2695177217","https://openalex.org/W2701059868","https://openalex.org/W2745146355","https://openalex.org/W2749572357","https://openalex.org/W2751592946","https://openalex.org/W2752586018","https://openalex.org/W2757528734","https://openalex.org/W2765200655","https://openalex.org/W2765986046","https://openalex.org/W2767011015","https://openalex.org/W2767079719","https://openalex.org/W2767094836","https://openalex.org/W2771446126","https://openalex.org/W2788572835","https://openalex.org/W2790316935","https://openalex.org/W2790393363","https://openalex.org/W2790572611","https://openalex.org/W2791445945","https://openalex.org/W2792020740","https://openalex.org/W2792648564","https://openalex.org/W2794001933","https://openalex.org/W2794215546","https://openalex.org/W2794601036","https://openalex.org/W2794787475","https://openalex.org/W2794855529","https://openalex.org/W2794995912","https://openalex.org/W2884943453","https://openalex.org/W2886689407","https://openalex.org/W2886694146","https://openalex.org/W2888125116","https://openalex.org/W2889746123","https://openalex.org/W2890010784","https://openalex.org/W2890206858","https://openalex.org/W2890356659","https://openalex.org/W2890471546","https://openalex.org/W2890559797","https://openalex.org/W2890697686","https://openalex.org/W2890991187","https://openalex.org/W2891250288","https://openalex.org/W2891316582","https://openalex.org/W2891407075","https://openalex.org/W2891621711","https://openalex.org/W2891810898","https://openalex.org/W2891865791","https://openalex.org/W2892187814","https://openalex.org/W2897830718","https://openalex.org/W2897865027","https://openalex.org/W2900962383","https://openalex.org/W2911730644","https://openalex.org/W2923323049","https://openalex.org/W2925389046","https://openalex.org/W2925709178","https://openalex.org/W2926178846","https://openalex.org/W2929264149","https://openalex.org/W2930110112","https://openalex.org/W2931153881","https://openalex.org/W2932329902","https://openalex.org/W2932551155","https://openalex.org/W2933426098","https://openalex.org/W2934843808","https://openalex.org/W2935152277","https://openalex.org/W2941048077","https://openalex.org/W2944196828","https://openalex.org/W2946363484","https://openalex.org/W2946864464","https://openalex.org/W2946905885","https://openalex.org/W2947081899","https://openalex.org/W2947103291","https://openalex.org/W2947243436","https://openalex.org/W2947292651","https://openalex.org/W2947499652","https://openalex.org/W2951914319","https://openalex.org/W2952330491","https://openalex.org/W2962736666","https://openalex.org/W2962763344","https://openalex.org/W2963058500","https://openalex.org/W2963077926","https://openalex.org/W2963378725","https://openalex.org/W2963408280","https://openalex.org/W2963539945","https://openalex.org/W2963674831","https://openalex.org/W2963704216","https://openalex.org/W2963721962","https://openalex.org/W2963742154","https://openalex.org/W2963822870","https://openalex.org/W2963844355","https://openalex.org/W2963954089","https://openalex.org/W2964097210","https://openalex.org/W2964261135","https://openalex.org/W2964294337","https://openalex.org/W2968084349","https://openalex.org/W2970044827","https://openalex.org/W2973629179","https://openalex.org/W2983060509","https://openalex.org/W2983277056","https://openalex.org/W2983941230","https://openalex.org/W2984260944","https://openalex.org/W2984488829","https://openalex.org/W2984515059","https://openalex.org/W2984998244","https://openalex.org/W2985527074","https://openalex.org/W2985580374","https://openalex.org/W2985610488","https://openalex.org/W2985766404","https://openalex.org/W2986159792","https://openalex.org/W2986164614","https://openalex.org/W2987577970","https://openalex.org/W2988337058","https://openalex.org/W2989358546","https://openalex.org/W2990138404","https://openalex.org/W2997006027","https://openalex.org/W3007037833","https://openalex.org/W3007121986","https://openalex.org/W3007224029","https://openalex.org/W3007409674","https://openalex.org/W3007469197","https://openalex.org/W3007623546","https://openalex.org/W3008176860","https://openalex.org/W3008439737","https://openalex.org/W3008443984","https://openalex.org/W3008748509","https://openalex.org/W3014111715","https://openalex.org/W3015248254","https://openalex.org/W3015326774","https://openalex.org/W3015535799","https://openalex.org/W3015584356","https://openalex.org/W3015693366","https://openalex.org/W3015975623","https://openalex.org/W3015990833","https://openalex.org/W3019195326","https://openalex.org/W3023153742","https://openalex.org/W3024103409","https://openalex.org/W3027379683","https://openalex.org/W3046764764","https://openalex.org/W3071470454","https://openalex.org/W3081618117","https://openalex.org/W3087086069","https://openalex.org/W3093021001","https://openalex.org/W3095325295","https://openalex.org/W3096214574","https://openalex.org/W3096264229","https://openalex.org/W3096609273","https://openalex.org/W3096706494","https://openalex.org/W3096738375","https://openalex.org/W3097730806","https://openalex.org/W3099264032","https://openalex.org/W3099608705","https://openalex.org/W3101243405","https://openalex.org/W3103553961","https://openalex.org/W3104102522","https://openalex.org/W3104158743","https://openalex.org/W3104224589","https://openalex.org/W3104454153","https://openalex.org/W3105920175","https://openalex.org/W3106646114","https://openalex.org/W3106873467","https://openalex.org/W3106934761","https://openalex.org/W3107015477","https://openalex.org/W3107488247","https://openalex.org/W3109104051","https://openalex.org/W3109937020","https://openalex.org/W3117449869","https://openalex.org/W3125645198","https://openalex.org/W3128341305","https://openalex.org/W3136098199","https://openalex.org/W3136177841","https://openalex.org/W3136284412","https://openalex.org/W3136550168","https://openalex.org/W3136918966","https://openalex.org/W3137646671","https://openalex.org/W3138076532","https://openalex.org/W3138147846","https://openalex.org/W3138153888","https://openalex.org/W3138173041","https://openalex.org/W3138227691","https://openalex.org/W3138348090","https://openalex.org/W3138596485","https://openalex.org/W3139208355","https://openalex.org/W3152700969","https://openalex.org/W3152758407","https://openalex.org/W3152901657","https://openalex.org/W3153103063","https://openalex.org/W3153453329","https://openalex.org/W3154528133","https://openalex.org/W3154581591","https://openalex.org/W3154803439","https://openalex.org/W3155666085","https://openalex.org/W3155680838","https://openalex.org/W3155915750","https://openalex.org/W3164055163","https://openalex.org/W3164146938","https://openalex.org/W3164936723","https://openalex.org/W3165639541","https://openalex.org/W3189812816","https://openalex.org/W3192357745","https://openalex.org/W3196012972","https://openalex.org/W3198109319","https://openalex.org/W3207651366","https://openalex.org/W3209752568","https://openalex.org/W3210951978","https://openalex.org/W3211484264","https://openalex.org/W3211487307","https://openalex.org/W3211707810","https://openalex.org/W3211832054","https://openalex.org/W3211853936","https://openalex.org/W3211930400","https://openalex.org/W3212077589","https://openalex.org/W3212276929","https://openalex.org/W3212419298","https://openalex.org/W3212502694","https://openalex.org/W3212523187","https://openalex.org/W3212677680","https://openalex.org/W3212969377","https://openalex.org/W3212981375","https://openalex.org/W3213008925","https://openalex.org/W3213232997","https://openalex.org/W3213350640","https://openalex.org/W3213462624","https://openalex.org/W3213537051","https://openalex.org/W3213725621","https://openalex.org/W3213807399","https://openalex.org/W3213814821","https://openalex.org/W3213849916","https://openalex.org/W3214009246","https://openalex.org/W3214522776","https://openalex.org/W3214714397","https://openalex.org/W3214716770","https://openalex.org/W3217268473","https://openalex.org/W4221155126","https://openalex.org/W4225433071","https://openalex.org/W4225476734","https://openalex.org/W4225526044","https://openalex.org/W4225859703","https://openalex.org/W4226079995","https://openalex.org/W4226086258","https://openalex.org/W4226096495","https://openalex.org/W4226201626","https://openalex.org/W4226272105","https://openalex.org/W4226283591","https://openalex.org/W4226466613","https://openalex.org/W4226536925","https://openalex.org/W4226550712","https://openalex.org/W4230625563","https://openalex.org/W4245852543","https://openalex.org/W4287117265","https://openalex.org/W4287391919","https://openalex.org/W4287600763","https://openalex.org/W4287644588","https://openalex.org/W4287671431","https://openalex.org/W4287690648","https://openalex.org/W4287723774","https://openalex.org/W4288028426","https://openalex.org/W4288057697","https://openalex.org/W4288057700","https://openalex.org/W4288057709","https://openalex.org/W4288057716","https://openalex.org/W4288057732","https://openalex.org/W4288057741","https://openalex.org/W4288057742","https://openalex.org/W4288057755","https://openalex.org/W4288057770","https://openalex.org/W4288057781","https://openalex.org/W4288057793","https://openalex.org/W4288057803","https://openalex.org/W4288057808","https://openalex.org/W4288086177","https://openalex.org/W4288094087","https://openalex.org/W4288107203","https://openalex.org/W4289891214","https://openalex.org/W4290960278","https://openalex.org/W4293783406","https://openalex.org/W4296423445","https://openalex.org/W4296544621","https://openalex.org/W4296562799","https://openalex.org/W4296596384","https://openalex.org/W4297632117","https://openalex.org/W4299337115","https://openalex.org/W4300824008","https://openalex.org/W4301726735","https://openalex.org/W4302305514","https://openalex.org/W4307003864","https://openalex.org/W4308338624","https://openalex.org/W4308381123","https://openalex.org/W4308391439","https://openalex.org/W4308391489","https://openalex.org/W4308410302","https://openalex.org/W4308632256","https://openalex.org/W4308632257","https://openalex.org/W4308632271","https://openalex.org/W4308632297","https://openalex.org/W4308632305","https://openalex.org/W4308641868","https://openalex.org/W4308641907","https://openalex.org/W4308642068","https://openalex.org/W4308642081","https://openalex.org/W4308642407","https://openalex.org/W4308642595","https://openalex.org/W4308642762","https://openalex.org/W4308642974","https://openalex.org/W4308643005","https://openalex.org/W4308643089","https://openalex.org/W4308643121","https://openalex.org/W4308643689","https://openalex.org/W4308644332","https://openalex.org/W4308644392","https://openalex.org/W4319990295","https://openalex.org/W6600175564","https://openalex.org/W6600234944","https://openalex.org/W6600297362","https://openalex.org/W6600310816","https://openalex.org/W6600319451","https://openalex.org/W6600373086","https://openalex.org/W6600424091","https://openalex.org/W6600561556","https://openalex.org/W6600586173","https://openalex.org/W6600654476","https://openalex.org/W6600655691","https://openalex.org/W6600688380","https://openalex.org/W6600962175","https://openalex.org/W6601215536","https://openalex.org/W6601626606","https://openalex.org/W6602054047","https://openalex.org/W6602845370","https://openalex.org/W6606674134","https://openalex.org/W6606742274","https://openalex.org/W6608544398","https://openalex.org/W6630140425","https://openalex.org/W6815619692","https://openalex.org/W6819521833"],"related_works":["https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W2140154150","https://openalex.org/W638612878","https://openalex.org/W4324138256","https://openalex.org/W2083330954"],"abstract_inverted_index":{"Reproducibility":[0],"is":[1,146],"crucial":[2],"to":[3,40,47,224],"the":[4,17,25,36,57,73,87,104,118,152,159],"advancement":[5],"of":[6,19,28,72,76,103,128,154,161,174],"science;":[7],"it":[8],"strengthens":[9],"confidence":[10],"in":[11,51,108,112,165,203,211,227,230],"seemingly":[12],"contradictory":[13],"results":[14],"and":[15,45,69,130,139,157],"expands":[16],"boundaries":[18],"known":[20],"discoveries.":[21],"Computer":[22,231],"Security":[23,58,232],"has":[24,60,78],"natural":[26],"benefit":[27],"creating":[29],"artifacts":[30,93,155,176],"that":[31,144,177,189,219],"should":[32],"facilitate":[33],"computational":[34,228],"reproducibility,":[35,66],"ability":[37],"for":[38,200],"others":[39],"use":[41],"someone":[42],"else's":[43],"code":[44],"data":[46],"independently":[48],"recreate":[49],"results,":[50,175],"a":[52,184],"relatively":[53],"straightforward":[54],"fashion.":[55],"While":[56],"community":[59],"recently":[61],"increased":[62],"its":[63],"attention":[64],"on":[65,98,171],"an":[67],"independent":[68],"comprehensive":[70],"measurement":[71,126],"current":[74],"state":[75],"reproducibility":[77,132,202,229],"not":[79],"been":[80],"conducted.":[81],"In":[82,214],"this":[83,180],"paper,":[84],"we":[85,196,217],"perform":[86,124],"first":[88],"such":[89],"study,":[90],"targeting":[91],"reproducible":[92],"generated":[94],"specifically":[95],"by":[96],"papers":[97],"machine":[99],"learning":[100],"security":[101,115],"(one":[102],"most":[105],"popular":[106],"areas":[107],"academic":[109],"research)":[110],"published":[111],"Tier":[113,166],"1":[114,167],"conferences":[116],"over":[117,133],"past":[119],"ten":[120],"years":[121,173],"(2013-2022).":[122],"We":[123],"our":[125,193,204,212],"study":[127],"indirect":[129],"direct":[131],"nearly":[134],"750":[135],"papers,":[136],"their":[137],"codebases,":[138],"datasets.":[140],"Our":[141],"analysis":[142],"shows":[143],"there":[145],"no":[147],"statistically":[148],"significant":[149,220],"difference":[150],"between":[151],"availability":[153],"before":[156],"after":[158],"introduction":[160],"Artifact":[162],"Evaluation":[163],"Committees":[164],"conferences.":[168],"However,":[169],"based":[170],"three":[172],"pass":[178],"through":[179],"process":[181],"work":[182],"at":[183],"higher":[185],"rate":[186],"than":[187],"those":[188],"do":[190],"not.":[191],"From":[192],"collected":[194],"findings,":[195],"offer":[197],"data-driven":[198],"suggestions":[199],"improving":[201],"community,":[205],"including":[206],"five":[207],"common":[208],"problems":[209],"observed":[210],"study.":[213],"so":[215],"doing,":[216],"demonstrate":[218],"progress":[221],"still":[222],"needs":[223],"be":[225],"made":[226],"research.":[233]},"counts_by_year":[{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":6}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}