{"id":"https://openalex.org/W4386833453","doi":"https://doi.org/10.1007/s10664-023-10341-8","title":"On the validity of retrospective predictive performance evaluation procedures in just-in-time software defect prediction","display_name":"On the validity of retrospective predictive performance evaluation procedures in just-in-time software defect prediction","publication_year":2023,"publication_date":"2023-09-01","ids":{"openalex":"https://openalex.org/W4386833453","doi":"https://doi.org/10.1007/s10664-023-10341-8"},"language":"en","primary_location":{"id":"doi:10.1007/s10664-023-10341-8","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-023-10341-8","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-023-10341-8.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10664-023-10341-8.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032643292","display_name":"Liyan Song","orcid":"https://orcid.org/0000-0003-1172-8825"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liyan Song","raw_affiliation_strings":["Guangdong Provincial Key Laboratory of Brain-inspired Intelligent Computation, Department of Computer Science and Engineering, Southern University of Science and Technology, Shenzhen, China","Research Institute of Trustworthy Autonomous Systems, Southern University of Science and Technology, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Guangdong Provincial Key Laboratory of Brain-inspired Intelligent Computation, Department of Computer Science and Engineering, Southern University of Science and Technology, Shenzhen, China","institution_ids":["https://openalex.org/I3045169105"]},{"raw_affiliation_string":"Research Institute of Trustworthy Autonomous Systems, Southern University of Science and Technology, Shenzhen, China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064089961","display_name":"Leandro L. Minku","orcid":"https://orcid.org/0000-0002-2639-0671"},"institutions":[{"id":"https://openalex.org/I79619799","display_name":"University of Birmingham","ror":"https://ror.org/03angcq70","country_code":"GB","type":"education","lineage":["https://openalex.org/I79619799"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Leandro L. Minku","raw_affiliation_strings":["School of Computer Science, The University of Birmingham, Edgbaston, Birmingham, B15 2TT, UK"],"affiliations":[{"raw_affiliation_string":"School of Computer Science, The University of Birmingham, Edgbaston, Birmingham, B15 2TT, UK","institution_ids":["https://openalex.org/I79619799"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100635494","display_name":"Xin Yao","orcid":"https://orcid.org/0000-0001-8837-4442"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Yao","raw_affiliation_strings":["Guangdong Provincial Key Laboratory of Brain-inspired Intelligent Computation, Department of Computer Science and Engineering, Southern University of Science and Technology, Shenzhen, China","Research Institute of Trustworthy Autonomous Systems, Southern University of Science and Technology, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Guangdong Provincial Key Laboratory of Brain-inspired Intelligent Computation, Department of Computer Science and Engineering, Southern University of Science and Technology, Shenzhen, China","institution_ids":["https://openalex.org/I3045169105"]},{"raw_affiliation_string":"Research Institute of Trustworthy Autonomous Systems, Southern University of Science and Technology, Shenzhen, China","institution_ids":["https://openalex.org/I3045169105"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5064089961"],"corresponding_institution_ids":["https://openalex.org/I79619799"],"apc_list":{"value":2290,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2290,"currency":"EUR","value_usd":2890},"fwci":3.1651,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.92976037,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"28","issue":"5","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6783281564712524},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.6319408416748047},{"id":"https://openalex.org/keywords/affect","display_name":"Affect (linguistics)","score":0.5771079063415527},{"id":"https://openalex.org/keywords/predictive-validity","display_name":"Predictive validity","score":0.4946083724498749},{"id":"https://openalex.org/keywords/software-bug","display_name":"Software bug","score":0.48935240507125854},{"id":"https://openalex.org/keywords/verification-and-validation","display_name":"Verification and validation","score":0.48902377486228943},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.4591670632362366},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.4166376292705536},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.33040833473205566},{"id":"https://openalex.org/keywords/operations-management","display_name":"Operations management","score":0.19620206952095032},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1859149932861328},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.13040560483932495},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.11106133460998535},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.09941726922988892},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07634484767913818}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6783281564712524},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.6319408416748047},{"id":"https://openalex.org/C2776035688","wikidata":"https://www.wikidata.org/wiki/Q1606558","display_name":"Affect (linguistics)","level":2,"score":0.5771079063415527},{"id":"https://openalex.org/C20685875","wikidata":"https://www.wikidata.org/wiki/Q7239678","display_name":"Predictive validity","level":2,"score":0.4946083724498749},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.48935240507125854},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.48902377486228943},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4591670632362366},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.4166376292705536},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.33040833473205566},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.19620206952095032},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1859149932861328},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.13040560483932495},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.11106133460998535},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.09941726922988892},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07634484767913818},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s10664-023-10341-8","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-023-10341-8","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-023-10341-8.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s10664-023-10341-8","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-023-10341-8","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-023-10341-8.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.5199999809265137,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[{"id":"https://openalex.org/G1121271761","display_name":null,"funder_award_id":"Program","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1231421488","display_name":null,"funder_award_id":"under","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1308942226","display_name":null,"funder_award_id":"2020B121201001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1477544716","display_name":null,"funder_award_id":"Guangdong","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1714633238","display_name":null,"funder_award_id":"21201001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2444603483","display_name":null,"funder_award_id":"2020B121","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2979871447","display_name":null,"funder_award_id":"2020B121201001","funder_id":"https://openalex.org/F4320327098","funder_display_name":"Guangdong Key Laboratory of Fermentation and Enzyme Engineering"},{"id":"https://openalex.org/G3065577720","display_name":null,"funder_award_id":"2017ZT07X386","funder_id":"https://openalex.org/F4320333334","funder_display_name":"Guangdong Province Introduction of Innovative R&D Team"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G391238517","display_name":null,"funder_award_id":", and","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7726157001","display_name":null,"funder_award_id":"Grant No.","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8399751032","display_name":null,"funder_award_id":"62002148","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8435032178","display_name":null,"funder_award_id":"2017ZT07X386","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8555519613","display_name":null,"funder_award_id":"62250710682","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320327098","display_name":"Guangdong Key Laboratory of Fermentation and Enzyme Engineering","ror":null},{"id":"https://openalex.org/F4320333334","display_name":"Guangdong Province Introduction of Innovative R&D Team","ror":null}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386833453.pdf"},"referenced_works_count":48,"referenced_works":["https://openalex.org/W154870643","https://openalex.org/W1569909748","https://openalex.org/W1605695115","https://openalex.org/W1904826605","https://openalex.org/W1980455221","https://openalex.org/W1984769753","https://openalex.org/W1987963388","https://openalex.org/W2007705030","https://openalex.org/W2052300809","https://openalex.org/W2057826716","https://openalex.org/W2073256825","https://openalex.org/W2082183626","https://openalex.org/W2100196534","https://openalex.org/W2107327607","https://openalex.org/W2118978333","https://openalex.org/W2120457925","https://openalex.org/W2126166995","https://openalex.org/W2146335723","https://openalex.org/W2147386665","https://openalex.org/W2148143831","https://openalex.org/W2149783794","https://openalex.org/W2150733339","https://openalex.org/W2150874999","https://openalex.org/W2157353183","https://openalex.org/W2158744032","https://openalex.org/W2172232422","https://openalex.org/W2242800359","https://openalex.org/W2276400542","https://openalex.org/W2408181256","https://openalex.org/W2530824252","https://openalex.org/W2604756720","https://openalex.org/W2606150376","https://openalex.org/W2796283679","https://openalex.org/W2954276596","https://openalex.org/W2955232136","https://openalex.org/W2955991060","https://openalex.org/W2957754705","https://openalex.org/W2964279097","https://openalex.org/W3011013751","https://openalex.org/W3089692094","https://openalex.org/W3105867435","https://openalex.org/W3173291597","https://openalex.org/W3175995826","https://openalex.org/W3213259750","https://openalex.org/W4226148285","https://openalex.org/W4236586490","https://openalex.org/W4243379839","https://openalex.org/W4285229502"],"related_works":["https://openalex.org/W2090249529","https://openalex.org/W2123460498","https://openalex.org/W1571231229","https://openalex.org/W2406746345","https://openalex.org/W234065253","https://openalex.org/W2971103475","https://openalex.org/W270826027","https://openalex.org/W2798306226","https://openalex.org/W2146844712","https://openalex.org/W142885039"],"abstract_inverted_index":{"Abstract":[0],"Just-In-Time":[1],"Software":[2],"Defect":[3],"Prediction":[4],"(JIT-SDP)":[5],"is":[6,104],"concerned":[7],"with":[8,29,153],"predicting":[9],"whether":[10,107],"software":[11,24,42,64,100,122,236,242],"changes":[12,25,43,123],"are":[13,215],"defect-inducing":[14],"or":[15],"clean.":[16],"It":[17,103],"operates":[18],"in":[19,32,128,203,212,218],"scenarios":[20],"where":[21],"labels":[22,50,62],"of":[23,63,84,95,121,135,138,142,150,157,169,184,220,231,248],"arrive":[26],"over":[27],"time":[28,37,55,159,186],"delay,":[30],"which":[31,154],"part":[33],"corresponds":[34],"to":[35,40,80,90],"the":[36,60,82,92,99,133,136,147,151,155,167,182,194,208],"we":[38,179],"wait":[39],"label":[41,70,163],"as":[44,125],"clean":[45,49],"(waiting":[46],"time).":[47],"However,":[48],"decided":[51],"based":[52],"on":[53,118,175,193],"waiting":[54,158,185,200,222],"may":[56,68],"be":[57,69],"different":[58,221],"from":[59],"true":[61],"changes,":[65],"i.e.,":[66],"there":[67],"noise.":[71],"This":[72],"typically":[73],"overlooked":[74],"issue":[75,109],"has":[76],"recently":[77],"been":[78],"shown":[79],"affect":[81,113,166],"validity":[83,134,168,195],"continuous":[85],"performance":[86,94,171,247],"evaluation":[87,114,172],"procedures":[88,115],"used":[89],"monitor":[91],"predictive":[93,210],"JIT-SDP":[96,129,213,249],"models":[97],"during":[98],"development":[101],"process.":[102],"still":[105],"unknown":[106],"this":[108],"could":[110],"potentially":[111],"also":[112,239],"that":[116,181,197],"rely":[117],"retrospective":[119,170],"collection":[120],"such":[124],"those":[126],"adopted":[127],"research":[130],"studies,":[131],"affecting":[132],"conclusions":[137],"a":[139,190],"large":[140],"body":[141],"existing":[143],"work.":[144],"We":[145],"conduct":[146],"first":[148],"investigation":[149],"extent":[152],"choice":[156,183],"and":[160,196,224],"its":[161],"corresponding":[162],"noise":[164],"would":[165],"procedures.":[173],"Based":[174],"13":[176],"GitHub":[177],"projects,":[178],"found":[180],"did":[187],"not":[188,232],"have":[189],"significant":[191],"impact":[192],"even":[198],"small":[199],"times":[201],"resulted":[202],"high":[204],"validity.":[205],"Therefore,":[206],"(1)":[207],"estimated":[209],"performances":[211],"studies":[214,227],"likely":[216],"reliable":[217],"view":[219],"times,":[223],"(2)":[225],"future":[226],"can":[228],"make":[229],"use":[230],"only":[233],"larger":[234],"(5k+":[235],"changes),":[237],"but":[238],"smaller":[240],"(1k":[241],"changes)":[243],"projects":[244],"for":[245],"evaluating":[246],"models.":[250]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4}],"updated_date":"2026-04-13T07:58:08.660418","created_date":"2025-10-10T00:00:00"}