{"id":"https://openalex.org/W4400519535","doi":"https://doi.org/10.1145/3641525.3663618","title":"Statistical comparison in empirical computer science with minimal computation usage","display_name":"Statistical comparison in empirical computer science with minimal computation usage","publication_year":2024,"publication_date":"2024-06-18","ids":{"openalex":"https://openalex.org/W4400519535","doi":"https://doi.org/10.1145/3641525.3663618"},"language":"en","primary_location":{"id":"doi:10.1145/3641525.3663618","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641525.3663618","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641525.3663618?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3641525.3663618?download=true","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012711650","display_name":"Timoth\u00e9e Mathieu","orcid":"https://orcid.org/0009-0006-3628-174X"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I2279609970","display_name":"Universit\u00e9 de Lille","ror":"https://ror.org/02kzqn938","country_code":"FR","type":"education","lineage":["https://openalex.org/I2279609970"]},{"id":"https://openalex.org/I4387153239","display_name":"Centre de Recherche en Informatique, Signal et Automatique de Lille","ror":"https://ror.org/05vrs3189","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I2279609970","https://openalex.org/I4387153239","https://openalex.org/I7454413"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Timoth\u00e9e Mathieu","raw_affiliation_strings":["Scool research group, INRIA, France and CRIStAL, Universit\u00e9 de Lille, France"],"raw_orcid":"https://orcid.org/0009-0006-3628-174X","affiliations":[{"raw_affiliation_string":"Scool research group, INRIA, France and CRIStAL, Universit\u00e9 de Lille, France","institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I2279609970","https://openalex.org/I4387153239"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103325479","display_name":"Philippe Preux","orcid":"https://orcid.org/0000-0002-2067-2838"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I2279609970","display_name":"Universit\u00e9 de Lille","ror":"https://ror.org/02kzqn938","country_code":"FR","type":"education","lineage":["https://openalex.org/I2279609970"]},{"id":"https://openalex.org/I4387153239","display_name":"Centre de Recherche en Informatique, Signal et Automatique de Lille","ror":"https://ror.org/05vrs3189","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I2279609970","https://openalex.org/I4387153239","https://openalex.org/I7454413"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Philippe Preux","raw_affiliation_strings":["CRIStAL, Universite de Lille, France and Scool research group, INRIA, France"],"raw_orcid":"https://orcid.org/0000-0002-2067-2838","affiliations":[{"raw_affiliation_string":"CRIStAL, Universite de Lille, France and Scool research group, INRIA, France","institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I2279609970","https://openalex.org/I4387153239"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.13894618,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"20","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.9839000105857849,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7369646430015564},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.6316779255867004},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3749697208404541},{"id":"https://openalex.org/keywords/computational-science","display_name":"Computational science","score":0.32588475942611694},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.32388341426849365},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.22210624814033508}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7369646430015564},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6316779255867004},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3749697208404541},{"id":"https://openalex.org/C459310","wikidata":"https://www.wikidata.org/wiki/Q117801","display_name":"Computational science","level":1,"score":0.32588475942611694},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32388341426849365},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.22210624814033508}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3641525.3663618","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641525.3663618","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641525.3663618?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},{"id":"pmh:oai:HAL:hal-04718314v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-04718314","pdf_url":"https://inria.hal.science/hal-04718314v1/document","source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM REP '24: ACM Conference on Reproducibility and Replicability, 35, ACM, pp.20-24, 2024, ⟨10.1145/3641525.3663618⟩","raw_type":"Proceedings"},{"id":"pmh:oai:lilloa.univ-lille.fr:20.500.12210/117563","is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.12210/117563","pdf_url":null,"source":{"id":"https://openalex.org/S4306402203","display_name":"LillOA (Universit\u00e9 de Lille (University Of Lille))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210123514","host_organization_name":"Centre d'Etudes en Civilisations, Langues et Litt\u00e9ratures Etrang\u00e8res","host_organization_lineage":["https://openalex.org/I4210123514"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.1145/3641525.3663618","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3641525.3663618","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3641525.3663618?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd ACM Conference on Reproducibility and Replicability","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.5899999737739563,"display_name":"Quality Education"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320316137","display_name":"Universit\u00e9 de Lille","ror":"https://ror.org/02kzqn938"},{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4400519535.pdf","grobid_xml":"https://content.openalex.org/works/W4400519535.grobid-xml"},"referenced_works_count":8,"referenced_works":["https://openalex.org/W2096166399","https://openalex.org/W2103001366","https://openalex.org/W2394625346","https://openalex.org/W2589849966","https://openalex.org/W2765082293","https://openalex.org/W3003257820","https://openalex.org/W4283721498","https://openalex.org/W6894201702"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W1429949169"],"abstract_inverted_index":{"The":[0,59],"replicability":[1,22],"of":[2,19,23,33,61,91,93,150,167],"computational":[3,38],"experiments":[4],"remains":[5],"a":[6,56,76,119,157,164],"fundamental":[7],"question.":[8],"For":[9],"example,":[10],"the":[11,20,31,62,89,116,143],"machine":[12],"learning":[13,130],"community":[14],"has":[15,124],"recently":[16,77],"become":[17],"aware":[18],"poor":[21],"many":[24],"experimental":[25],"studies":[26,140],"that":[27,47,104,112,154,159],"aim":[28],"at":[29],"comparing":[30],"performance":[32],"various":[34],"algorithms.":[35],"Due":[36],"to":[37,44,54,96,109,141],"costs,":[39],"it":[40,155],"is":[41,75,107,156],"often":[42],"necessary":[43],"use":[45,144],"methods":[46],"require":[48],"as":[49,52,98,100],"few":[50],"computations":[51],"possible":[53,101],"obtain":[55],"replicable":[57,67],"conclusion.":[58],"conclusion":[60],"comparison":[63],"should":[64],"also":[65],"be":[66,161],"which":[68],"calls":[69],"for":[70],"appropriate":[71],"statistical":[72,79],"tests.":[73,86],"AdaStop":[74,87,123,145],"introduced":[78],"test":[80,158],"based":[81],"on":[82,128,163],"multiple":[83],"group":[84],"sequential":[85],"adapts":[88],"number":[90],"executions":[92],"each":[94],"experiment":[95],"stop":[97],"early":[99],"while":[102],"ensuring":[103],"enough":[105],"information":[106],"available":[108],"distinguish":[110],"algorithms":[111],"perform":[113],"better":[114],"than":[115],"others":[117],"in":[118],"statistically":[120],"significant":[121],"way.":[122],"been":[125],"initially":[126],"exemplified":[127],"reinforcement":[129],"tasks.":[131],"In":[132],"this":[133],"short":[134],"paper,":[135],"we":[136],"consider":[137],"3":[138],"case":[139],"investigate":[142],"beyond":[146],"its":[147],"original":[148],"field":[149],"application,":[151],"and":[152],"demonstrate":[153],"may":[160],"used":[162],"wide":[165],"range":[166],"application":[168],"domains.":[169]},"counts_by_year":[],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}