{"id":"https://openalex.org/W4393459063","doi":"https://doi.org/10.5281/zenodo.6266675","title":"Supplementary Material for \"Evaluating Classifiers in SE Research: The ECSER Pipeline and Two Replication Studies\"","display_name":"Supplementary Material for \"Evaluating Classifiers in SE Research: The ECSER Pipeline and Two Replication Studies\"","publication_year":2021,"publication_date":"2021-09-03","ids":{"openalex":"https://openalex.org/W4393459063","doi":"https://doi.org/10.5281/zenodo.6266675"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.6266675","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.6266675","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"dataset"},"type":"dataset","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.6266675","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063228362","display_name":"Davide Dell\u2019Anna","orcid":"https://orcid.org/0000-0002-1162-8341"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Davide Dell'Anna","raw_affiliation_strings":["Delft University of Technology"],"raw_orcid":"https://orcid.org/0000-0002-1162-8341","affiliations":[{"raw_affiliation_string":"Delft University of Technology","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024981898","display_name":"Fatma Ba\u015fak Aydemir","orcid":"https://orcid.org/0000-0003-3833-3997"},"institutions":[{"id":"https://openalex.org/I4405392","display_name":"Bo\u011fazi\u00e7i University","ror":"https://ror.org/03z9tma90","country_code":"TR","type":"education","lineage":["https://openalex.org/I4405392"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Fatma Ba\u015fak Aydemir","raw_affiliation_strings":["Bo\u011fazi\u00e7i University"],"raw_orcid":"https://orcid.org/0000-0003-3833-3997","affiliations":[{"raw_affiliation_string":"Bo\u011fazi\u00e7i University","institution_ids":["https://openalex.org/I4405392"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069677848","display_name":"Fabiano Dalpiaz","orcid":"https://orcid.org/0000-0003-4480-3887"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Fabiano Dalpiaz","raw_affiliation_strings":["Utrecht University"],"raw_orcid":"https://orcid.org/0000-0003-4480-3887","affiliations":[{"raw_affiliation_string":"Utrecht University","institution_ids":["https://openalex.org/I193662353"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5063228362"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.07559999823570251,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.07559999823570251,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.06030000001192093,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13398","display_name":"Data Analysis with R","score":0.0340999998152256,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.8929963111877441},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6105988025665283},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.47913262248039246},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36069023609161377},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.21381062269210815},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.13818225264549255},{"id":"https://openalex.org/keywords/virology","display_name":"Virology","score":0.0813782811164856}],"concepts":[{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.8929963111877441},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6105988025665283},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.47913262248039246},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36069023609161377},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.21381062269210815},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13818225264549255},{"id":"https://openalex.org/C159047783","wikidata":"https://www.wikidata.org/wiki/Q7215","display_name":"Virology","level":1,"score":0.0813782811164856}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5281/zenodo.6266675","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.6266675","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"dataset"}],"best_oa_location":{"id":"doi:10.5281/zenodo.6266675","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.6266675","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"dataset"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W4205713785","https://openalex.org/W2001405890","https://openalex.org/W3016766501","https://openalex.org/W2901033488","https://openalex.org/W2382290278","https://openalex.org/W2478288626"],"abstract_inverted_index":{"This":[0,64],"supplementary":[1,130],"material":[2],"for":[3,72,91,116,147,275,308],"the":[4,35,56,60,73,87,92,103,112,117,126,129,134,139,142,145,156,166,173,186,189,196,203,206,210,215,230,234,251],"article\"Evaluating":[5],"Classifiers":[6,277],"in":[7,41,155,165,183,248,278],"SE":[8,249,279],"Research:":[9,280],"The":[10,31,83,108,159,179,199,281],"ECSER":[11,70,252,282],"Pipeline":[12,283],"and":[13,22,45,77,89,99,114,209,233,243,254,284,298,318],"Two":[14,285],"Replication":[15,286],"Studies.":[16],"Dell\u2019Anna,":[17,238],"D.;":[18],"Aydemir,":[19,242,268],"F.":[20,24,300],"B.;":[21],"Dalpiaz,":[23],"Empirical":[25,258],"Software":[26,259,331],"Engineering.":[27],"2022\"":[28],"includes":[29,172],"ECSER-ExploratoryStudy.csv:":[30],"annotated":[32],"meta-data":[33],"of":[34,59,69,94,119,128,205,304],"papers":[36],"that":[37,51,85,110,150,175],"have":[38],"been":[39],"published":[40],"ICSE":[42,333],"between":[43],"2019":[44],"2021.":[46,320,338],"ECSER_ROCplots+StatTest.ipynb:":[47],"A":[48],"python":[49,170],"notebook":[50,171,200],"compares":[52,202],"classifiers":[53,149,204,247],"adn":[54],"checks":[55],"statistical":[57],"significance":[58],"comparison":[61],"results.":[62],"ECSER_SummaryOfReplicationSteps.pdf:":[63],"table":[65],"presents":[66,125],"a":[67],"summary":[68],"steps":[71],"two":[74,255],"original":[75,207],"studies":[76],"our":[78],"applications":[79],"on":[80,102,330],"ECSER.":[81],"ECSER_RE:":[82],"directory":[84,109],"holds":[86,111],"datasets":[88],"code":[90,113,140,146,174],"replication":[93,118,256],"Hay":[95],"et":[96,121],"al.":[97,122],"[1]":[98],"additional":[100],"runs":[101,176],"new":[104],"data":[105,115,161,190],"sets.":[106],"ECSER_FF:":[107],"Alshammari":[120],"[2]":[123],"README.md":[124],"structure":[127],"materials.":[131],"requirements.txt":[132],"lists":[133],"dependencies":[135],"needed":[136],"to":[137,219],"run":[138],"In":[141,185],"ECSER_RE":[143],"directory,":[144,188],"multiple":[148],"are":[151,153,163,181,192,212],"compared":[152],"kept":[154,213],"\"Classifiers\"":[157],"directory.":[158,168,198,217],"public":[160],"sets":[162,191],"shared":[164],"Datasets":[167],"\"ECSER_RE_Compare_Classifiers.ipynb\"":[169],"each":[177],"classifier.":[178],"results":[180,211],"presented":[182,193],"\"ECSER_RE_results-Promise-vs-all.csv\".":[184],"ECSER_FF":[187],"directly":[194],"under":[195,214],"main":[197],"\"ECSER-FF-Compare_Classifiers.ipynb\"":[201],"study":[208],"\"ECSER_FF_results\"":[216],"How":[218],"cite":[220,229],"this":[221,226],"repository":[222],"If":[223],"you":[224],"use":[225],"repository,":[227,235],"please":[228],"reference":[231],"paper,":[232],"as":[236],"below:":[237],"Davide,":[239],"Fatma":[240,266],"Ba\u015fak":[241,267],"Fabiano":[244,270],"Dalpiaz.":[245,271],"\"Evaluating":[246,276],"research:":[250],"pipeline":[253],"studies.\"":[257],"Engineering":[260],"28.1":[261],"(2023):":[262],"3.":[263],"Davide":[264],"Dell'Anna,":[265],"&":[269],"(2021).":[272],"Supplementary":[273],"Material":[274],"Studies\"":[287],"[Data":[288],"set].":[289],"Zenodo.":[290],"https://doi.org/10.5281/zenodo.6266675":[291],"Tobias":[292],"Hey,":[293],"Jan":[294],"Keim,":[295],"Anne":[296],"Koziolek,":[297],"Walter":[299],"Tichy.":[301],"2020.":[302],"SupplementaryMaterial":[303],"\"NoRBERT:":[305],"Transfer":[306],"Learning":[307],"Requirements":[309],"Classification\".":[310],"https://doi.org/10.5281/zenodo.3874137":[311],"Abdulrahman":[312],"Alshammari,":[313],"Christopher":[314],"Morris,":[315],"Michael":[316],"Hilton,":[317],"JonathanBell.":[319],"FlakeFlagger:":[321],"Predicting":[322],"Flakiness":[323],"Without":[324],"Rerunning":[325],"Tests.":[326],"In43rdIEEE/ACM":[327],"International":[328],"Conference":[329],"Engineering,":[332],"2021,":[334],"Madrid,Spain,":[335],"22-30":[336],"May":[337],"IEEE,":[339],"1572\u20131584.":[340],"https://doi.org/10.1109/ICSE43902.2021.00140":[341]},"counts_by_year":[{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}