{"id":"https://openalex.org/W4390482179","doi":"https://doi.org/10.1109/seaa60479.2023.00022","title":"Early Stopping of Non-productive Performance Testing Experiments Using Measurement Mutations","display_name":"Early Stopping of Non-productive Performance Testing Experiments Using Measurement Mutations","publication_year":2023,"publication_date":"2023-09-06","ids":{"openalex":"https://openalex.org/W4390482179","doi":"https://doi.org/10.1109/seaa60479.2023.00022"},"language":"en","primary_location":{"id":"doi:10.1109/seaa60479.2023.00022","is_oa":false,"landing_page_url":"https://doi.org/10.1109/seaa60479.2023.00022","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 49th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032152788","display_name":"Milad Abdullah","orcid":"https://orcid.org/0000-0002-0696-6354"},"institutions":[{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Milad Abdullah","raw_affiliation_strings":["Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","institution_ids":["https://openalex.org/I21250087"]},{"raw_affiliation_string":"Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic","institution_ids":["https://openalex.org/I21250087"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074547557","display_name":"Lubom\u00edr Bulej","orcid":"https://orcid.org/0000-0002-4573-6084"},"institutions":[{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Lubom\u00edr Bulej","raw_affiliation_strings":["Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","institution_ids":["https://openalex.org/I21250087"]},{"raw_affiliation_string":"Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic","institution_ids":["https://openalex.org/I21250087"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082418720","display_name":"Tom\u00e1\u0161 Bure\u0161","orcid":"https://orcid.org/0000-0003-3622-9918"},"institutions":[{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Tom\u00e1\u0161 Bure\u0161","raw_affiliation_strings":["Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","institution_ids":["https://openalex.org/I21250087"]},{"raw_affiliation_string":"Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic","institution_ids":["https://openalex.org/I21250087"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020532214","display_name":"Vojt\u011bch Hork\u00fd","orcid":"https://orcid.org/0000-0003-2146-160X"},"institutions":[{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Vojt\u011bch Hork\u00fd","raw_affiliation_strings":["Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","institution_ids":["https://openalex.org/I21250087"]},{"raw_affiliation_string":"Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic","institution_ids":["https://openalex.org/I21250087"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101719359","display_name":"Petr T\u016fma","orcid":"https://orcid.org/0000-0002-7035-2322"},"institutions":[{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Petr T\u016fma","raw_affiliation_strings":["Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Charles University,Faculty of Mathematics and Physics,Prague,Czech Republic","institution_ids":["https://openalex.org/I21250087"]},{"raw_affiliation_string":"Faculty of Mathematics and Physics, Charles University, Prague, Czech Republic","institution_ids":["https://openalex.org/I21250087"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I21250087"],"apc_list":null,"apc_paid":null,"fwci":1.0299,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.75174024,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"86","last_page":"93"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.9444684982299805},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.9220578670501709},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7615952491760254},{"id":"https://openalex.org/keywords/software-performance-testing","display_name":"Software performance testing","score":0.5431228876113892},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5347028970718384},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.5200414657592773},{"id":"https://openalex.org/keywords/performance-improvement","display_name":"Performance improvement","score":0.4451114237308502},{"id":"https://openalex.org/keywords/software-system","display_name":"Software system","score":0.17495188117027283},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.09082898497581482},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08606404066085815},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.08011144399642944}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.9444684982299805},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.9220578670501709},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7615952491760254},{"id":"https://openalex.org/C178059732","wikidata":"https://www.wikidata.org/wiki/Q1982529","display_name":"Software performance testing","level":5,"score":0.5431228876113892},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5347028970718384},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.5200414657592773},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.4451114237308502},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.17495188117027283},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.09082898497581482},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08606404066085815},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.08011144399642944},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/seaa60479.2023.00022","is_oa":false,"landing_page_url":"https://doi.org/10.1109/seaa60479.2023.00022","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 49th Euromicro Conference on Software Engineering and Advanced Applications (SEAA)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.5,"id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320334678","display_name":"European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1909800943","https://openalex.org/W1965377045","https://openalex.org/W2053135018","https://openalex.org/W2259042472","https://openalex.org/W2277866467","https://openalex.org/W2614734074","https://openalex.org/W2934326692","https://openalex.org/W2954796040","https://openalex.org/W2959321909","https://openalex.org/W2979993677","https://openalex.org/W2991561094","https://openalex.org/W3003877692","https://openalex.org/W3005281198","https://openalex.org/W3007460877","https://openalex.org/W3101119896","https://openalex.org/W3108371183","https://openalex.org/W3123858332","https://openalex.org/W3126244934","https://openalex.org/W3155432886","https://openalex.org/W3208695041","https://openalex.org/W3208874162","https://openalex.org/W4235975582","https://openalex.org/W4285804579","https://openalex.org/W4286239570","https://openalex.org/W4307789634","https://openalex.org/W4316657727","https://openalex.org/W4320068688","https://openalex.org/W4393429176","https://openalex.org/W6902123020","https://openalex.org/W7028761299"],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W4399363378","https://openalex.org/W2010147257","https://openalex.org/W2086898575","https://openalex.org/W2155330023","https://openalex.org/W2400069262","https://openalex.org/W4390482179"],"abstract_inverted_index":{"Modern":[0],"software":[1,122],"projects":[2],"often":[3],"incorporate":[4],"some":[5],"form":[6],"of":[7,67,77,88,121,144,155],"performance":[8,19,37,78,97,159],"testing":[9,25],"into":[10],"their":[11],"development":[12],"cycle,":[13],"intending":[14],"to":[15,46,94,116,151],"detect":[16,36,95],"changes":[17,38,123],"in":[18,50,99],"between":[20],"commits":[21],"or":[22],"releases.":[23],"Performance":[24],"generally":[26],"relies":[27],"on":[28,108,124,131],"experimental":[29],"evaluation":[30],"using":[31],"various":[32],"benchmark":[33,70,125],"workloads.":[34],"To":[35,73],"reliably,":[39],"benchmarks":[40],"must":[41],"be":[42],"executed":[43],"many":[44],"times":[45],"account":[47],"for":[48,84],"variability":[49],"the":[51,65,75,85,118,129,132,145,156],"measurement":[52],"results.":[53],"While":[54],"considered":[55],"best":[56],"practice,":[57],"this":[58],"approach":[59,83,130],"can":[60,140,149],"become":[61],"prohibitively":[62],"expensive":[63],"when":[64],"number":[66],"versions":[68],"and":[69,136],"workloads":[71],"increases.":[72],"alleviate":[74],"cost":[76],"testing,":[79],"we":[80,148],"propose":[81],"an":[82],"early":[86],"stopping":[87,104],"non-productive":[89],"experiments":[90,146],"that":[91,138],"are":[92,106],"unlikely":[93],"a":[96,100],"bug":[98],"particular":[101],"benchmark.":[102],"The":[103],"conditions":[105],"based":[107],"benchmark-specific":[109],"thresholds":[110],"determined":[111],"from":[112],"historical":[113],"data":[114],"modified":[115],"emulate":[117],"potential":[119],"effects":[120],"performance.":[126],"We":[127],"evaluate":[128],"GraalVM":[133],"benchmarking":[134],"project":[135],"show":[137],"it":[139],"eliminate":[141],"about":[142,153],"50%":[143],"if":[147],"afford":[150],"ignore":[152],"15%":[154],"least":[157],"significant":[158],"changes.":[160]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-27T14:10:00.468798","created_date":"2025-10-10T00:00:00"}