{"id":"https://openalex.org/W4308518385","doi":"https://doi.org/10.1007/s10664-022-10243-1","title":"Evaluating classifiers in SE research: the ECSER pipeline and two replication studies","display_name":"Evaluating classifiers in SE research: the ECSER pipeline and two replication studies","publication_year":2022,"publication_date":"2022-11-08","ids":{"openalex":"https://openalex.org/W4308518385","doi":"https://doi.org/10.1007/s10664-022-10243-1"},"language":"en","primary_location":{"id":"doi:10.1007/s10664-022-10243-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-022-10243-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-022-10243-1.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10664-022-10243-1.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063228362","display_name":"Davide Dell\u2019Anna","orcid":"https://orcid.org/0000-0002-1162-8341"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Davide Dell\u2019Anna","raw_affiliation_strings":["Delft University of Technology, Delft, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, The Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024981898","display_name":"Fatma Ba\u015fak Aydemir","orcid":"https://orcid.org/0000-0003-3833-3997"},"institutions":[{"id":"https://openalex.org/I4405392","display_name":"Bo\u011fazi\u00e7i University","ror":"https://ror.org/03z9tma90","country_code":"TR","type":"education","lineage":["https://openalex.org/I4405392"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Fatma Ba\u015fak Aydemir","raw_affiliation_strings":["Bo\u01e7azi\u00e7i University, Istanbul, Turkey"],"affiliations":[{"raw_affiliation_string":"Bo\u01e7azi\u00e7i University, Istanbul, Turkey","institution_ids":["https://openalex.org/I4405392"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069677848","display_name":"Fabiano Dalpiaz","orcid":"https://orcid.org/0000-0003-4480-3887"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Fabiano Dalpiaz","raw_affiliation_strings":["Utrecht University, Utrecht, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Utrecht University, Utrecht, The Netherlands","institution_ids":["https://openalex.org/I193662353"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5063228362"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":{"value":2290,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2290,"currency":"EUR","value_usd":2890},"fwci":6.0472,"has_fulltext":true,"cited_by_count":19,"citation_normalized_percentile":{"value":0.96491387,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"28","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.767109751701355},{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.7518394589424133},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7361424565315247},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5874857902526855},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5530562400817871},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.531168520450592},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.46562477946281433},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4071110188961029},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.39971691370010376},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3430001735687256},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.14494389295578003}],"concepts":[{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.767109751701355},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.7518394589424133},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7361424565315247},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5874857902526855},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5530562400817871},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.531168520450592},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.46562477946281433},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4071110188961029},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.39971691370010376},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3430001735687256},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.14494389295578003},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1007/s10664-022-10243-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-022-10243-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-022-10243-1.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:tudelft.nl:uuid:236561ad-b070-4acb-8553-21ddb021b04c","is_oa":false,"landing_page_url":"http://resolver.tudelft.nl/uuid:236561ad-b070-4acb-8553-21ddb021b04c","pdf_url":null,"source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"journal article"},{"id":"pmh:uu:oai:dspace.library.uu.nl:1874/426572","is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/426572","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Empirical Software Engineering, 28(1), 1. Springer Netherlands","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1007/s10664-022-10243-1","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-022-10243-1","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-022-10243-1.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.4300000071525574}],"awards":[{"id":"https://openalex.org/G5327710781","display_name":null,"funder_award_id":"BIDEB 2232 grant 118C255","funder_id":"https://openalex.org/F4320322626","funder_display_name":"T\u00fcrkiye Bilimsel ve Teknolojik Ara\u015ft\u0131rma Kurumu"},{"id":"https://openalex.org/G62546385","display_name":null,"funder_award_id":"BIDEB 2232","funder_id":"https://openalex.org/F4320322626","funder_display_name":"T\u00fcrkiye Bilimsel ve Teknolojik Ara\u015ft\u0131rma Kurumu"},{"id":"https://openalex.org/G728939322","display_name":null,"funder_award_id":"118C255","funder_id":"https://openalex.org/F4320322626","funder_display_name":"T\u00fcrkiye Bilimsel ve Teknolojik Ara\u015ft\u0131rma Kurumu"},{"id":"https://openalex.org/G8042658919","display_name":null,"funder_award_id":"BIDEB","funder_id":"https://openalex.org/F4320322626","funder_display_name":"T\u00fcrkiye Bilimsel ve Teknolojik Ara\u015ft\u0131rma Kurumu"}],"funders":[{"id":"https://openalex.org/F4320322626","display_name":"T\u00fcrkiye Bilimsel ve Teknolojik Ara\u015ft\u0131rma Kurumu","ror":"https://ror.org/04w9kkr77"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4308518385.pdf","grobid_xml":"https://content.openalex.org/works/W4308518385.grobid-xml"},"referenced_works_count":70,"referenced_works":["https://openalex.org/W4214443","https://openalex.org/W49973284","https://openalex.org/W161653996","https://openalex.org/W1554638674","https://openalex.org/W1746353556","https://openalex.org/W1981276685","https://openalex.org/W1999798506","https://openalex.org/W1999954155","https://openalex.org/W2000031724","https://openalex.org/W2009086942","https://openalex.org/W2038455038","https://openalex.org/W2046830558","https://openalex.org/W2107026277","https://openalex.org/W2108821163","https://openalex.org/W2112865076","https://openalex.org/W2122399740","https://openalex.org/W2122987719","https://openalex.org/W2127626360","https://openalex.org/W2132887549","https://openalex.org/W2133990480","https://openalex.org/W2143637886","https://openalex.org/W2151666086","https://openalex.org/W2158698691","https://openalex.org/W2161948305","https://openalex.org/W2167490420","https://openalex.org/W2181845779","https://openalex.org/W2344072768","https://openalex.org/W2424693453","https://openalex.org/W2484065175","https://openalex.org/W2562251009","https://openalex.org/W2565032187","https://openalex.org/W2611471088","https://openalex.org/W2613159689","https://openalex.org/W2735864825","https://openalex.org/W2753180476","https://openalex.org/W2758253540","https://openalex.org/W2795013017","https://openalex.org/W2801952332","https://openalex.org/W2954093663","https://openalex.org/W2954227696","https://openalex.org/W2963203256","https://openalex.org/W2963520355","https://openalex.org/W2993303751","https://openalex.org/W3003877886","https://openalex.org/W3009290003","https://openalex.org/W3017201198","https://openalex.org/W3017317785","https://openalex.org/W3026094172","https://openalex.org/W3049429073","https://openalex.org/W3080999680","https://openalex.org/W3086800606","https://openalex.org/W3091299963","https://openalex.org/W3091973425","https://openalex.org/W3104736511","https://openalex.org/W3135371071","https://openalex.org/W3194114348","https://openalex.org/W3200707343","https://openalex.org/W3211007339","https://openalex.org/W4211072535","https://openalex.org/W4234556776","https://openalex.org/W4235970556","https://openalex.org/W4239672721","https://openalex.org/W4245055982","https://openalex.org/W4246413330","https://openalex.org/W4250023757","https://openalex.org/W4285670724","https://openalex.org/W4291213652","https://openalex.org/W4393421629","https://openalex.org/W4393459063","https://openalex.org/W6702248584"],"related_works":["https://openalex.org/W4205713785","https://openalex.org/W3016766501","https://openalex.org/W2901033488","https://openalex.org/W2031325922","https://openalex.org/W3086458689","https://openalex.org/W4287666211","https://openalex.org/W3037187668","https://openalex.org/W2898161671","https://openalex.org/W2249859249","https://openalex.org/W4251387359"],"abstract_inverted_index":{"Abstract":[0],"Context":[1],"Automated":[2],"classifiers,":[3],"often":[4],"based":[5],"on":[6],"machine":[7],"learning":[8],"(ML),":[9],"are":[10,101],"increasingly":[11],"used":[12],"in":[13,71,109,138,156,171,175],"software":[14,176],"engineering":[15,173],"(SE)":[16],"for":[17,59,65,149],"labelling":[18],"previously":[19],"unseen":[20],"SE":[21,66],"data.":[22],"Researchers":[23],"have":[24],"proposed":[25],"automated":[26,153],"classifiers":[27],"that":[28,100],"predict":[29],"if":[30,37,44],"a":[31,35,38,48,103,145],"code":[32],"chunk":[33],"is":[34,40,51,117],"clone,":[36],"requirement":[39],"functional":[41],"or":[42,96],"non-functional,":[43],"the":[45,88,107,134,183,186,188,206,220],"outcome":[46],"of":[47,57,106,114,126,147,185,216],"test":[49],"case":[50],"non-deterministic,":[52],"etc.":[53],"Objective":[54],"The":[55,112],"lack":[56],"guidelines":[58],"applying":[60],"and":[61,85,87,128,151,174,200],"reporting":[62,129],"classification":[63,154],"techniques":[64],"research":[67,74,121,155,170],"leads":[68],"to":[69,118,168,181],"studies":[70,163,190],"which":[72,143],"important":[73],"steps":[75,148],"may":[76,90],"be":[77,83],"skipped,":[78],"key":[79],"findings":[80,203,218],"might":[81],"not":[82,102,195],"identified":[84,204],"shared,":[86],"readers":[89],"find":[91],"reported":[92],"results":[93],"(e.g.,":[94],"precision":[95],"recall":[97],"above":[98],"90%)":[99],"credible":[104],"representation":[105],"performance":[108],"operational":[110],"contexts.":[111],"goal":[113],"this":[115],"paper":[116],"advance":[119],"ML4SE":[120],"by":[122,205],"proposing":[123],"rigorous":[124],"ways":[125],"conducting":[127,150],"research.":[130],"Results":[131],"We":[132],"introduce":[133],"ECSER":[135,167,192],"(Evaluating":[136],"Classifiers":[137],"Software":[139],"Engineering":[140],"Research)":[141],"pipeline,":[142,187],"includes":[144],"series":[146],"evaluating":[152],"SE.":[157],"Then,":[158],"we":[159,165,198,210],"conduct":[160],"two":[161],"replication":[162,189],"where":[164],"apply":[166],"recent":[169],"requirements":[172],"testing.":[177],"Conclusions":[178],"In":[179],"addition":[180],"demonstrating":[182],"applicability":[184],"demonstrate":[191],"\u2019s":[193],"usefulness:":[194],"only":[196],"do":[197],"confirm":[199],"strengthen":[201],"some":[202],"original":[207,221],"authors,":[208],"but":[209],"also":[211],"discover":[212],"additional":[213],"ones.":[214,222],"Some":[215],"these":[217],"contradict":[219]},"counts_by_year":[{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":4}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}