{"id":"https://openalex.org/W4406874869","doi":"https://doi.org/10.1145/3711119","title":"A Comprehensive Survey of Benchmarks for Improvement of Software's Non-Functional Properties","display_name":"A Comprehensive Survey of Benchmarks for Improvement of Software's Non-Functional Properties","publication_year":2025,"publication_date":"2025-01-27","ids":{"openalex":"https://openalex.org/W4406874869","doi":"https://doi.org/10.1145/3711119"},"language":"en","primary_location":{"id":"doi:10.1145/3711119","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711119","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3711119","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043338869","display_name":"Aymeric Blot","orcid":"https://orcid.org/0000-0003-0485-5279"},"institutions":[{"id":"https://openalex.org/I2802487185","display_name":"Universit\u00e9 du littoral c\u00f4te d'opale","ror":"https://ror.org/02gdcg342","country_code":"FR","type":"education","lineage":["https://openalex.org/I2802487185"]},{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]},{"id":"https://openalex.org/I56067802","display_name":"Universit\u00e9 de Rennes","ror":"https://ror.org/015m7wh34","country_code":"FR","type":"education","lineage":["https://openalex.org/I56067802"]}],"countries":["FR","GB"],"is_corresponding":false,"raw_author_name":"Aymeric Blot","raw_affiliation_strings":["University College London, London United Kingdom of Great Britain and Northern Ireland","Universit\u00e9 de Rennes, Rennes, France","Universit\u00e9 du Littoral C\u00f4te d'Opale, Calais France","UCL - University College of London [London] (Gower Street, London WC1E 6BT - United Kingdom)","Universit\u00e9 du Littoral C\u00f4te d'Opale,  Calais France","DiverSe - Diversity-centric Software Engineering (Campus de Beaulieu, 35042 Rennes cedex - France)"],"raw_orcid":"https://orcid.org/0000-0003-0485-5279","affiliations":[{"raw_affiliation_string":"University College London, London United Kingdom of Great Britain and Northern Ireland","institution_ids":["https://openalex.org/I45129253"]},{"raw_affiliation_string":"Universit\u00e9 de Rennes, Rennes, France","institution_ids":["https://openalex.org/I56067802"]},{"raw_affiliation_string":"Universit\u00e9 du Littoral C\u00f4te d'Opale, Calais France","institution_ids":["https://openalex.org/I2802487185"]},{"raw_affiliation_string":"UCL - University College of London [London] (Gower Street, London WC1E 6BT - United Kingdom)","institution_ids":["https://openalex.org/I45129253"]},{"raw_affiliation_string":"Universit\u00e9 du Littoral C\u00f4te d'Opale,  Calais France","institution_ids":["https://openalex.org/I2802487185"]},{"raw_affiliation_string":"DiverSe - Diversity-centric Software Engineering (Campus de Beaulieu, 35042 Rennes cedex - France)","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065281209","display_name":"Justyna Petke","orcid":"https://orcid.org/0000-0002-7833-6044"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Justyna Petke","raw_affiliation_strings":["University College London, London United Kingdom of Great Britain and Northern Ireland","UCL - University College of London [London] (Gower Street, London WC1E 6BT - United Kingdom)"],"raw_orcid":"https://orcid.org/0000-0002-7833-6044","affiliations":[{"raw_affiliation_string":"University College London, London United Kingdom of Great Britain and Northern Ireland","institution_ids":["https://openalex.org/I45129253"]},{"raw_affiliation_string":"UCL - University College of London [London] (Gower Street, London WC1E 6BT - United Kingdom)","institution_ids":["https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.5204,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.88917896,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"57","issue":"7","first_page":"1","last_page":"36"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12238","display_name":"Green IT and Sustainability","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8580297231674194},{"id":"https://openalex.org/keywords/javascript","display_name":"JavaScript","score":0.6874998807907104},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5974530577659607},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.5261943936347961},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5190032720565796},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.5060681700706482},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4581649899482727},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.44779253005981445},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3413654863834381},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.14194437861442566}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8580297231674194},{"id":"https://openalex.org/C544833334","wikidata":"https://www.wikidata.org/wiki/Q2005","display_name":"JavaScript","level":2,"score":0.6874998807907104},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5974530577659607},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.5261943936347961},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5190032720565796},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5060681700706482},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4581649899482727},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.44779253005981445},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3413654863834381},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.14194437861442566},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3711119","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711119","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:10203326","is_oa":true,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/10203326/","pdf_url":"https://discovery.ucl.ac.uk/10203326/1/main.pdf","source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM Computing Surveys (2025) (In press).","raw_type":"Article"},{"id":"pmh:oai:HAL:hal-04936383v1","is_oa":true,"landing_page_url":"https://hal.science/hal-04936383","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM Computing Surveys, 2025, 57 (7), pp.1-36 / Article n\u00b0168. ⟨10.1145/3711119⟩","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1145/3711119","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711119","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8100000023841858,"display_name":"Affordable and clean energy","id":"https://metadata.un.org/sdg/7"}],"awards":[{"id":"https://openalex.org/G2073061185","display_name":"Automated Software Specialisation Using Genetic Improvement","funder_award_id":"EP/P023991/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":149,"referenced_works":["https://openalex.org/W1527396576","https://openalex.org/W1782174992","https://openalex.org/W1810396090","https://openalex.org/W1880498397","https://openalex.org/W1964031104","https://openalex.org/W1966533006","https://openalex.org/W1969127567","https://openalex.org/W1969951522","https://openalex.org/W1977461164","https://openalex.org/W1979866694","https://openalex.org/W1984074188","https://openalex.org/W1985866831","https://openalex.org/W1992851788","https://openalex.org/W1993909392","https://openalex.org/W2001363041","https://openalex.org/W2024024375","https://openalex.org/W2035210248","https://openalex.org/W2040046786","https://openalex.org/W2040257309","https://openalex.org/W2045218056","https://openalex.org/W2048936605","https://openalex.org/W2058821752","https://openalex.org/W2069278717","https://openalex.org/W2071110673","https://openalex.org/W2072386027","https://openalex.org/W2080756777","https://openalex.org/W2083774192","https://openalex.org/W2088628452","https://openalex.org/W2095985365","https://openalex.org/W2096648046","https://openalex.org/W2103705372","https://openalex.org/W2103837157","https://openalex.org/W2106683494","https://openalex.org/W2116863572","https://openalex.org/W2118134309","https://openalex.org/W2118518999","https://openalex.org/W2120845776","https://openalex.org/W2123059332","https://openalex.org/W2128090947","https://openalex.org/W2138007781","https://openalex.org/W2140418760","https://openalex.org/W2149399143","https://openalex.org/W2156504173","https://openalex.org/W2156723666","https://openalex.org/W2158737060","https://openalex.org/W2159756807","https://openalex.org/W2163385469","https://openalex.org/W2168397028","https://openalex.org/W2171392117","https://openalex.org/W2197075213","https://openalex.org/W2235765161","https://openalex.org/W2287872303","https://openalex.org/W2291190927","https://openalex.org/W2397792450","https://openalex.org/W2406013352","https://openalex.org/W2441512324","https://openalex.org/W2484802798","https://openalex.org/W2492658338","https://openalex.org/W2500074838","https://openalex.org/W2503699867","https://openalex.org/W2504358330","https://openalex.org/W2513616710","https://openalex.org/W2515029334","https://openalex.org/W2531444577","https://openalex.org/W2536899800","https://openalex.org/W2550527757","https://openalex.org/W2557652470","https://openalex.org/W2608579592","https://openalex.org/W2619922765","https://openalex.org/W2620561093","https://openalex.org/W2624813485","https://openalex.org/W2724793896","https://openalex.org/W2735662636","https://openalex.org/W2735684785","https://openalex.org/W2735771726","https://openalex.org/W2741175893","https://openalex.org/W2768116780","https://openalex.org/W2772612468","https://openalex.org/W2793681106","https://openalex.org/W2799953961","https://openalex.org/W2810235207","https://openalex.org/W2852727117","https://openalex.org/W2888445849","https://openalex.org/W2888699475","https://openalex.org/W2894917785","https://openalex.org/W2898527848","https://openalex.org/W2899348357","https://openalex.org/W2902657696","https://openalex.org/W2905303940","https://openalex.org/W2913592991","https://openalex.org/W2928669301","https://openalex.org/W2946042477","https://openalex.org/W2950680102","https://openalex.org/W2953634222","https://openalex.org/W2954215843","https://openalex.org/W2958727599","https://openalex.org/W2959020790","https://openalex.org/W2973282825","https://openalex.org/W2986132139","https://openalex.org/W2992223827","https://openalex.org/W2998660555","https://openalex.org/W3000784322","https://openalex.org/W3016562408","https://openalex.org/W3035580571","https://openalex.org/W3049161099","https://openalex.org/W3087577597","https://openalex.org/W3088310535","https://openalex.org/W3091836068","https://openalex.org/W3107855260","https://openalex.org/W3118757005","https://openalex.org/W3122888791","https://openalex.org/W3131150301","https://openalex.org/W3133585083","https://openalex.org/W3133912113","https://openalex.org/W3137412430","https://openalex.org/W3139827290","https://openalex.org/W3145959950","https://openalex.org/W3147258241","https://openalex.org/W3148880536","https://openalex.org/W3152851310","https://openalex.org/W3153920384","https://openalex.org/W3159783110","https://openalex.org/W3162900075","https://openalex.org/W3165613912","https://openalex.org/W3166825710","https://openalex.org/W3181319689","https://openalex.org/W3195186037","https://openalex.org/W3208407575","https://openalex.org/W3210474818","https://openalex.org/W3212295442","https://openalex.org/W4210721538","https://openalex.org/W4242955857","https://openalex.org/W4249737999","https://openalex.org/W4251235123","https://openalex.org/W4284680326","https://openalex.org/W4285004820","https://openalex.org/W4287814182","https://openalex.org/W4288057686","https://openalex.org/W4310652305","https://openalex.org/W4312245811","https://openalex.org/W4387360103","https://openalex.org/W4389990388","https://openalex.org/W4391827190","https://openalex.org/W4392774524","https://openalex.org/W4394660114","https://openalex.org/W6640755664","https://openalex.org/W6761200461","https://openalex.org/W6784261146","https://openalex.org/W6788293844"],"related_works":["https://openalex.org/W2341492732","https://openalex.org/W3187193180","https://openalex.org/W106542691","https://openalex.org/W2461078469","https://openalex.org/W1699080303","https://openalex.org/W4297799326","https://openalex.org/W3116064965","https://openalex.org/W2378211422","https://openalex.org/W123790205","https://openalex.org/W2085515337"],"abstract_inverted_index":{"Despite":[0],"recent":[1],"increase":[2],"in":[3,33,72,118,150],"research":[4,55,84,119],"on":[5,56,68,212],"improvement":[6,57,124,132,194],"of":[7,10,23,42,45,58,61,83,103,133,157,171,190],"non-functional":[8,59,134,193],"properties":[9,60,136],"software,":[11,62,170],"such":[12,27],"as":[13],"energy":[14],"usage":[15],"or":[16,199],"program":[17],"size,":[18],"there":[19],"is":[20,112,125,174],"a":[21,65,98],"lack":[22,189],"standard":[24],"benchmarks":[25,44,70,191],"for":[26,131,183,187,192],"work.":[28],"This":[29],"absence":[30],"hinders":[31],"progress":[32],"the":[34,40,69,73,113,151,158,203,206],"field,":[35],"and":[36,52,91,100,145],"raises":[37],"questions":[38],"about":[39],"representativeness":[41],"current":[43],"real-world":[46],"software.":[47],"To":[48],"address":[49],"these":[50],"issues":[51],"facilitate":[53],"further":[54],"we":[63],"conducted":[64],"comprehensive":[66],"survey":[67],"used":[71],"field":[74],"thus":[75],"far.":[76],"We":[77,107,179],"searched":[78],"five":[79],"major":[80],"online":[81],"repositories":[82],"work,":[85,185],"collecting":[86],"5,499":[87],"publications":[88],"(4,066":[89],"unique),":[90],"systematically":[92],"identified":[93,208],"relevant":[94,105],"papers":[95,160,209],"to":[96],"construct":[97],"rich":[99],"diverse":[101],"corpus":[102],"425":[104,159,207],"studies.":[106],"find":[108],"that":[109,163,195],"execution":[110],"time":[111],"most":[114,175],"frequently":[115],"improved":[116],"property":[117],"work":[120,162],"(63%),":[121],"while":[122],"multi-objective":[123],"rarely":[126],"considered":[127],"(7%).":[128],"Static":[129],"approaches":[130,142],"software":[135],"are":[137,210],"prevalent":[138],"(51%),":[139],"with":[140],"exploratory":[141],"(18%":[143],"evolutionary":[144],"15%":[146],"non-evolutionary)":[147],"increasingly":[148],"popular":[149,176],"last":[152],"10":[153],"years.":[154],"Only":[155],"39%":[156],"describe":[161],"uses":[164],"benchmark":[165],"suites,":[166],"rather":[167],"than":[168],"single":[169],"those":[172],"SPEC":[173],"(63":[177],"papers).":[178],"also":[180],"provide":[181],"recommendations":[182],"future":[184],"noting,":[186],"instance,":[188],"covers":[196],"Python,":[197],"JavaScript,":[198],"mobile":[200],"devices.":[201],"All":[202],"details":[204],"regarding":[205],"available":[211],"our":[213],"dedicated":[214],"webpage:":[215],"https://bloa.github.io/nfunc_survey":[216],".":[217]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-25T09:21:30.201066","created_date":"2025-01-28T00:00:00"}