{"id":"https://openalex.org/W4415381654","doi":"https://doi.org/10.1145/3736731.3746152","title":"Lessons Learned from Five Years of Artifact Evaluations at EuroSys","display_name":"Lessons Learned from Five Years of Artifact Evaluations at EuroSys","publication_year":2025,"publication_date":"2025-07-29","ids":{"openalex":"https://openalex.org/W4415381654","doi":"https://doi.org/10.1145/3736731.3746152"},"language":"en","primary_location":{"id":"doi:10.1145/3736731.3746152","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3736731.3746152","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3736731.3746152","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067397962","display_name":"Daniele Cono D\u2019Elia","orcid":"https://orcid.org/0000-0003-4358-976X"},"institutions":[{"id":"https://openalex.org/I861853513","display_name":"Sapienza University of Rome","ror":"https://ror.org/02be6w209","country_code":"IT","type":"education","lineage":["https://openalex.org/I861853513"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Daniele Cono D'Elia","raw_affiliation_strings":["Sapienza University of Rome, Rome, Italy"],"raw_orcid":"https://orcid.org/0000-0003-4358-976X","affiliations":[{"raw_affiliation_string":"Sapienza University of Rome, Rome, Italy","institution_ids":["https://openalex.org/I861853513"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033123165","display_name":"Thaleia Dimitra Doudali","orcid":"https://orcid.org/0000-0002-3197-839X"},"institutions":[{"id":"https://openalex.org/I4210162154","display_name":"IMDEA Software Institute","ror":"https://ror.org/04xvfkh51","country_code":"ES","type":"facility","lineage":["https://openalex.org/I105140100","https://openalex.org/I4210162154"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Thaleia Dimitra Doudali","raw_affiliation_strings":["IMDEA Software Institute, Madrid, Spain"],"raw_orcid":"https://orcid.org/0000-0002-3197-839X","affiliations":[{"raw_affiliation_string":"IMDEA Software Institute, Madrid, Spain","institution_ids":["https://openalex.org/I4210162154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083941826","display_name":"Cristiano Giuffrida","orcid":"https://orcid.org/0000-0002-8329-5929"},"institutions":[{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Cristiano Giuffrida","raw_affiliation_strings":["VU Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-8329-5929","affiliations":[{"raw_affiliation_string":"VU Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020461697","display_name":"Miguel Matos","orcid":"https://orcid.org/0000-0001-6916-2866"},"institutions":[{"id":"https://openalex.org/I121345201","display_name":"Instituto de Engenharia de Sistemas e Computadores Investiga\u00e7\u00e3o e Desenvolvimento","ror":"https://ror.org/04mqy3p58","country_code":"PT","type":"nonprofit","lineage":["https://openalex.org/I121345201","https://openalex.org/I4210125590"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Miguel Matos","raw_affiliation_strings":["IST Lisbon & INESC-ID, Lisbon, Portugal"],"raw_orcid":"https://orcid.org/0000-0001-6916-2866","affiliations":[{"raw_affiliation_string":"IST Lisbon & INESC-ID, Lisbon, Portugal","institution_ids":["https://openalex.org/I121345201"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065116578","display_name":"Mathias Payer","orcid":"https://orcid.org/0000-0001-5054-7547"},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Mathias Payer","raw_affiliation_strings":["EPFL, Lausanne, Switzerland"],"raw_orcid":"https://orcid.org/0000-0001-5054-7547","affiliations":[{"raw_affiliation_string":"EPFL, Lausanne, Switzerland","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048550148","display_name":"Solal Pirelli","orcid":"https://orcid.org/0009-0003-4336-1316"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Solal Pirelli","raw_affiliation_strings":["Independent Researcher, Lausanne, Switzerland"],"raw_orcid":"https://orcid.org/0009-0003-4336-1316","affiliations":[{"raw_affiliation_string":"Independent Researcher, Lausanne, Switzerland","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045292308","display_name":"Georgios Portokalidis","orcid":"https://orcid.org/0000-0002-1665-803X"},"institutions":[{"id":"https://openalex.org/I4210162154","display_name":"IMDEA Software Institute","ror":"https://ror.org/04xvfkh51","country_code":"ES","type":"facility","lineage":["https://openalex.org/I105140100","https://openalex.org/I4210162154"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Georgios Portokalidis","raw_affiliation_strings":["IMDEA Software Institute, Madrid, Spain"],"raw_orcid":"https://orcid.org/0000-0002-1665-803X","affiliations":[{"raw_affiliation_string":"IMDEA Software Institute, Madrid, Spain","institution_ids":["https://openalex.org/I4210162154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033418614","display_name":"Valerio Schiavoni","orcid":"https://orcid.org/0000-0003-1493-6603"},"institutions":[{"id":"https://openalex.org/I57825437","display_name":"University of Neuch\u00e2tel","ror":"https://ror.org/00vasag41","country_code":"CH","type":"education","lineage":["https://openalex.org/I57825437"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Valerio Schiavoni","raw_affiliation_strings":["University of Neuch\u00e2tel, Neuch\u00e2tel, Switzerland"],"raw_orcid":"https://orcid.org/0000-0003-1493-6603","affiliations":[{"raw_affiliation_string":"University of Neuch\u00e2tel, Neuch\u00e2tel, Switzerland","institution_ids":["https://openalex.org/I57825437"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000789304","display_name":"Salvatore Signorello","orcid":"https://orcid.org/0000-0001-6628-0541"},"institutions":[{"id":"https://openalex.org/I83558840","display_name":"Universidade Nova de Lisboa","ror":"https://ror.org/02xankh89","country_code":"PT","type":"education","lineage":["https://openalex.org/I83558840"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Salvatore Signorello","raw_affiliation_strings":["NOVA University Lisbon, Lisbon, Portugal"],"raw_orcid":"https://orcid.org/0000-0001-6628-0541","affiliations":[{"raw_affiliation_string":"NOVA University Lisbon, Lisbon, Portugal","institution_ids":["https://openalex.org/I83558840"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070365933","display_name":"Anjo Vahldiek-Oberwagner","orcid":"https://orcid.org/0000-0002-6983-0630"},"institutions":[{"id":"https://openalex.org/I4210094487","display_name":"Intel (Germany)","ror":"https://ror.org/00m2x0g47","country_code":"DE","type":"company","lineage":["https://openalex.org/I1343180700","https://openalex.org/I4210094487"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Anjo Vahldiek-Oberwagner","raw_affiliation_strings":["Intel Labs, Berlin, Germany"],"raw_orcid":"https://orcid.org/0000-0002-6983-0630","affiliations":[{"raw_affiliation_string":"Intel Labs, Berlin, Germany","institution_ids":["https://openalex.org/I4210094487"]}]}],"institutions":[],"countries_distinct_count":6,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5067397962"],"corresponding_institution_ids":["https://openalex.org/I861853513"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.378542,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"108","last_page":"120"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11949","display_name":"Nuclear Physics and Applications","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/3108","display_name":"Radiation"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11949","display_name":"Nuclear Physics and Applications","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/3108","display_name":"Radiation"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12039","display_name":"Electron and X-Ray Spectroscopy Techniques","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/2508","display_name":"Surfaces, Coatings and Films"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11216","display_name":"Radiation Detection and Scintillator Technologies","score":0.9607999920845032,"subfield":{"id":"https://openalex.org/subfields/3108","display_name":"Radiation"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/artifact","display_name":"Artifact (error)","score":0.611299991607666},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.2587999999523163},{"id":"https://openalex.org/keywords/good-practice","display_name":"Good practice","score":0.25279998779296875},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.22669999301433563}],"concepts":[{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.611299991607666},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4641000032424927},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35030001401901245},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.2587999999523163},{"id":"https://openalex.org/C3018028102","wikidata":"https://www.wikidata.org/wiki/Q254125","display_name":"Good practice","level":2,"score":0.25279998779296875},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.25270000100135803},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.25270000100135803},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.250900000333786},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.23389999568462372},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.22669999301433563}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3736731.3746152","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3736731.3746152","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},{"id":"pmh:oai:research.vu.nl:publications/3f52b121-a545-4e32-94b0-9290b160786f","is_oa":true,"landing_page_url":"https://research.vu.nl/en/publications/3f52b121-a545-4e32-94b0-9290b160786f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401107","display_name":"VU Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I865915315","host_organization_name":"Vrije Universiteit Amsterdam","host_organization_lineage":["https://openalex.org/I865915315"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"D\u2019Elia, D C, Doudali, T D, Giuffrida, C, Matos, M, Payer, M, Pirelli, S, Portokalidis, G, Schiavoni, V, Signorello, S & Vahldiek-Oberwagner, A 2025, Lessons Learned from Five Years of Artifact Evaluations at EuroSys. in ACM REP '25: Proceedings of the 3rd ACM Conference on Reproducibility and Replicability. Association for Computing Machinery, Inc, pp. 108-120, 3rd ACM Conference on Reproducibility and Replicability, ACM REP 2025, Vancouver, Canada, 29/07/25. https://doi.org/10.1145/3736731.3746152","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:research.vu.nl:openaire/3f52b121-a545-4e32-94b0-9290b160786f","is_oa":true,"landing_page_url":"https://hdl.handle.net/1871.1/3f52b121-a545-4e32-94b0-9290b160786f","pdf_url":null,"source":{"id":"https://openalex.org/S4306401107","display_name":"VU Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I865915315","host_organization_name":"Vrije Universiteit Amsterdam","host_organization_lineage":["https://openalex.org/I865915315"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"D\u2019Elia, D C, Doudali, T D, Giuffrida, C, Matos, M, Payer, M, Pirelli, S, Portokalidis, G, Schiavoni, V, Signorello, S & Vahldiek-Oberwagner, A 2025, Lessons Learned from Five Years of Artifact Evaluations at EuroSys. in ACM REP '25: Proceedings of the 3rd ACM Conference on Reproducibility and Replicability. Association for Computing Machinery, Inc, pp. 108-120, 3rd ACM Conference on Reproducibility and Replicability, ACM REP 2025, Vancouver, Canada, 29/07/25. https://doi.org/10.1145/3736731.3746152","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1145/3736731.3746152","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3736731.3746152","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W2082256810","https://openalex.org/W2161694107","https://openalex.org/W2220857351","https://openalex.org/W2283438976","https://openalex.org/W2406493898","https://openalex.org/W3044177754","https://openalex.org/W3099627437","https://openalex.org/W3128249766","https://openalex.org/W3179169088","https://openalex.org/W3211435516","https://openalex.org/W3216160250","https://openalex.org/W4281679534","https://openalex.org/W4283323680","https://openalex.org/W4396600580"],"related_works":[],"abstract_inverted_index":{"Artifact":[0],"Evaluation":[1],"(\u201cAE\u201d)":[2],"is":[3],"now":[4],"an":[5],"accepted":[6],"practice":[7],"in":[8,44,77],"the":[9,25,32,63],"systems":[10,59],"community.":[11],"However,":[12],"AE":[13,28,49,87],"processes":[14,29],"are":[15],"inconsistent":[16],"across":[17,21,35],"venues":[18,36],"and":[19,37,47,86],"even":[20],"different":[22],"editions":[23,54],"of":[24,55],"same":[26,33],"venue.":[27],"regularly":[30],"encounter":[31],"problems":[34],"years.":[38],"Based":[39],"on":[40],"our":[41],"collective":[42],"experience":[43],"chairing":[45],"various":[46],"heterogeneous":[48],"committees":[50],"for":[51,83],"five":[52],"consecutive":[53],"EuroSys,":[56],"a":[57],"large":[58],"conference,":[60],"we":[61,65],"present":[62],"challenges":[64,76],"believe":[66],"most":[67],"pressing.":[68],"We":[69],"propose":[70],"concrete":[71],"steps":[72],"to":[73],"address":[74],"these":[75],"future":[78,84],"AEs,":[79],"serving":[80],"as":[81],"guidelines":[82],"chairs":[85],"committees.":[88]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-22T00:00:00"}