{"id":"https://openalex.org/W4226148285","doi":"https://doi.org/10.1109/tse.2022.3158831","title":"A Procedure to Continuously Evaluate Predictive Performance of Just-In-Time Software Defect Prediction Models During Software Development","display_name":"A Procedure to Continuously Evaluate Predictive Performance of Just-In-Time Software Defect Prediction Models During Software Development","publication_year":2022,"publication_date":"2022-03-15","ids":{"openalex":"https://openalex.org/W4226148285","doi":"https://doi.org/10.1109/tse.2022.3158831"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2022.3158831","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2022.3158831","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research.birmingham.ac.uk/en/publications/0da8e0b3-c2ab-41df-803f-7de313054438","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032643292","display_name":"Liyan Song","orcid":"https://orcid.org/0000-0003-1172-8825"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liyan Song","raw_affiliation_strings":["Research Institute of Trustworthy Autonomous Systems, Southern University of Science and Technology, Shenzhen, Guangdong, China","Department of Computer Science and Engineering, Guangdong Provincial Key Laboratory of Brain-inspired Intelligent Computation, Southern University of Science and Technology, Shenzhen, Guangdong, China"],"raw_orcid":"https://orcid.org/0000-0003-1172-8825","affiliations":[{"raw_affiliation_string":"Research Institute of Trustworthy Autonomous Systems, Southern University of Science and Technology, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I3045169105"]},{"raw_affiliation_string":"Department of Computer Science and Engineering, Guangdong Provincial Key Laboratory of Brain-inspired Intelligent Computation, Southern University of Science and Technology, Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064089961","display_name":"Leandro L. Minku","orcid":"https://orcid.org/0000-0002-2639-0671"},"institutions":[{"id":"https://openalex.org/I79619799","display_name":"University of Birmingham","ror":"https://ror.org/03angcq70","country_code":"GB","type":"education","lineage":["https://openalex.org/I79619799"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Leandro L. Minku","raw_affiliation_strings":["School of Computer Science, University of Birmingham, Birmingham, U.K"],"raw_orcid":"https://orcid.org/0000-0002-2639-0671","affiliations":[{"raw_affiliation_string":"School of Computer Science, University of Birmingham, Birmingham, U.K","institution_ids":["https://openalex.org/I79619799"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.124,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.97086813,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"49","issue":"2","first_page":"646","last_page":"666"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7989816665649414},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.6495232582092285},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5748838186264038},{"id":"https://openalex.org/keywords/predictive-modelling","display_name":"Predictive modelling","score":0.524185836315155},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.500474214553833},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.49003517627716064},{"id":"https://openalex.org/keywords/software-bug","display_name":"Software bug","score":0.48808062076568604},{"id":"https://openalex.org/keywords/verification-and-validation","display_name":"Verification and validation","score":0.4425480365753174},{"id":"https://openalex.org/keywords/software-metric","display_name":"Software metric","score":0.4333062171936035},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.38969963788986206},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3823995292186737},{"id":"https://openalex.org/keywords/operations-management","display_name":"Operations management","score":0.11393928527832031},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.10254064202308655},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07420971989631653}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7989816665649414},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.6495232582092285},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5748838186264038},{"id":"https://openalex.org/C45804977","wikidata":"https://www.wikidata.org/wiki/Q7239673","display_name":"Predictive modelling","level":2,"score":0.524185836315155},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.500474214553833},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.49003517627716064},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.48808062076568604},{"id":"https://openalex.org/C48002344","wikidata":"https://www.wikidata.org/wiki/Q2919644","display_name":"Verification and validation","level":2,"score":0.4425480365753174},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.4333062171936035},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.38969963788986206},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3823995292186737},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.11393928527832031},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.10254064202308655},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07420971989631653}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tse.2022.3158831","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2022.3158831","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/0da8e0b3-c2ab-41df-803f-7de313054438","is_oa":true,"landing_page_url":"https://research.birmingham.ac.uk/en/publications/0da8e0b3-c2ab-41df-803f-7de313054438","pdf_url":null,"source":{"id":"https://openalex.org/S4306402634","display_name":"University of Birmingham Research Portal (University of Birmingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79619799","host_organization_name":"University of Birmingham","host_organization_lineage":["https://openalex.org/I79619799"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Song, L & Minku, L 2022, 'A procedure to continuously evaluate predictive performance of just-in-time software defect prediction models during software development', IEEE Transactions on Software Engineering. https://doi.org/10.1109/TSE.2022.3158831","raw_type":"article"},{"id":"pmh:oai:pure.atira.dk:publications/0da8e0b3-c2ab-41df-803f-7de313054438","is_oa":false,"landing_page_url":"https://birmingham.elsevierpure.com/en/publications/0da8e0b3-c2ab-41df-803f-7de313054438","pdf_url":null,"source":{"id":"https://openalex.org/S4306402634","display_name":"University of Birmingham Research Portal (University of Birmingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79619799","host_organization_name":"University of Birmingham","host_organization_lineage":["https://openalex.org/I79619799"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/0da8e0b3-c2ab-41df-803f-7de313054438","is_oa":true,"landing_page_url":"https://research.birmingham.ac.uk/en/publications/0da8e0b3-c2ab-41df-803f-7de313054438","pdf_url":null,"source":{"id":"https://openalex.org/S4306402634","display_name":"University of Birmingham Research Portal (University of Birmingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79619799","host_organization_name":"University of Birmingham","host_organization_lineage":["https://openalex.org/I79619799"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Song, L & Minku, L 2022, 'A procedure to continuously evaluate predictive performance of just-in-time software defect prediction models during software development', IEEE Transactions on Software Engineering. https://doi.org/10.1109/TSE.2022.3158831","raw_type":"article"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.5400000214576721,"id":"https://metadata.un.org/sdg/9"}],"awards":[{"id":"https://openalex.org/G1308942226","display_name":null,"funder_award_id":"2020B121201001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4781280766","display_name":null,"funder_award_id":"EP/R006660/2","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G521923880","display_name":null,"funder_award_id":"KQTD2016112514355531","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6192470683","display_name":"Stable Prediction of Defect-Inducing Software Changes (SPDISC)","funder_award_id":"EP/R006660/2","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G781164028","display_name":null,"funder_award_id":"R006660/2","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G800496194","display_name":null,"funder_award_id":"518055","funder_id":"https://openalex.org/F4320327605","funder_display_name":"Southern University of Science and Technology"},{"id":"https://openalex.org/G8399751032","display_name":null,"funder_award_id":"62002148","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8435032178","display_name":null,"funder_award_id":"2017ZT07X386","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320325571","display_name":"Shenzhen University","ror":"https://ror.org/01vy4gh70"},{"id":"https://openalex.org/F4320327605","display_name":"Southern University of Science and Technology","ror":null},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":40,"referenced_works":["https://openalex.org/W1605695115","https://openalex.org/W1904826605","https://openalex.org/W1984769753","https://openalex.org/W1987963388","https://openalex.org/W2003155177","https://openalex.org/W2007705030","https://openalex.org/W2052300809","https://openalex.org/W2057826716","https://openalex.org/W2073256825","https://openalex.org/W2082183626","https://openalex.org/W2084689094","https://openalex.org/W2100196534","https://openalex.org/W2107327607","https://openalex.org/W2118978333","https://openalex.org/W2126166995","https://openalex.org/W2147386665","https://openalex.org/W2148143831","https://openalex.org/W2150733339","https://openalex.org/W2150874999","https://openalex.org/W2172232422","https://openalex.org/W2276400542","https://openalex.org/W2408181256","https://openalex.org/W2483479423","https://openalex.org/W2530824252","https://openalex.org/W2604756720","https://openalex.org/W2606150376","https://openalex.org/W2796283679","https://openalex.org/W2847284300","https://openalex.org/W2955232136","https://openalex.org/W2955991060","https://openalex.org/W2964279097","https://openalex.org/W3216620463","https://openalex.org/W4235354960","https://openalex.org/W4236586490","https://openalex.org/W4239181501","https://openalex.org/W4243379839","https://openalex.org/W6752866118","https://openalex.org/W6783769817","https://openalex.org/W6910696474","https://openalex.org/W6929791901"],"related_works":["https://openalex.org/W4360910036","https://openalex.org/W2798385000","https://openalex.org/W1602996893","https://openalex.org/W2187488199","https://openalex.org/W4320067626","https://openalex.org/W2183508512","https://openalex.org/W3014006153","https://openalex.org/W2031021105","https://openalex.org/W4224250221","https://openalex.org/W3034143783"],"abstract_inverted_index":{"Just-In-Time":[0],"Software":[1],"Defect":[2],"Prediction":[3],"(JIT-SDP)":[4],"uses":[5],"machine":[6],"learning":[7],"to":[8,41,62,67,85,119,124,126,147,179,201,255],"predict":[9],"whether":[10,64,199],"software":[11,52,60,101,128,153,191],"changes":[12,20,102,129],"are":[13,285],"defect-inducing":[14],"or":[15,65],"clean.":[16],"When":[17],"adopting":[18],"JIT-SDP,":[19],"in":[21,90,116,215,239,277],"the":[22,30,44,51,69,120,148,152,159,174,190,195,207,232,237,269,290],"underlying":[23],"defect":[24],"generating":[25],"process":[26,54],"may":[27,144],"significantly":[28],"affect":[29],"predictive":[31,45,88,164,182,259],"performance":[32,46,89,165,183,212,242,260],"of":[33,47,56,151,161,177,184,198,209,250,258,279,289,292],"JIT-SDP":[34,48,169,185,272,281],"models":[35,49,74,186],"over":[36,75,110,187,261],"time.":[37,76,262],"Therefore,":[38],"being":[39],"able":[40],"continuously":[42,86,180],"track":[43],"during":[50,189],"development":[53,192],"is":[55,95],"utmost":[57],"importance":[58],"for":[59,107,168],"companies":[61],"decide":[63],"not":[66,96,145,266],"trust":[68],"predictions":[70],"provided":[71],"by":[72],"such":[73,93,210],"However,":[77],"there":[78],"has":[79],"been":[80],"little":[81],"discussion":[82,176],"on":[83,139,218,231],"how":[84,178],"evaluate":[87,181],"practice,":[91],"and":[92,194,200,246,274],"evaluation":[94,108,166,213],"straightforward.":[97],"In":[98],"particular,":[99],"labeled":[100],"that":[103,224],"can":[104,156,253],"be":[105],"used":[106],"arrive":[109],"time":[111,121,143,188,205,226,252],"with":[112],"a":[113,140,228],"delay,":[114],"which":[115,280],"part":[117],"corresponds":[118],"we":[122,222],"have":[123],"wait":[125],"label":[127,136,150],"as":[130],"\u2018clean\u2019":[131],"(waiting":[132],"time).":[133],"A":[134],"clean":[135],"assigned":[137],"based":[138],"given":[141],"waiting":[142,204,225,251,293],"correspond":[146],"true":[149],"changes.":[154],"This":[155,171],"potentially":[157],"hinder":[158],"validity":[160,208],"any":[162],"continuous":[163,211],"procedure":[167,214],"models.":[170],"paper":[172],"provides":[173],"first":[175,196],"process,":[193],"investigation":[197],"what":[202],"extent":[203],"affects":[206],"JIT-SDP.":[216],"Based":[217],"13":[219],"GitHub":[220],"projects,":[221],"found":[223],"had":[227],"significant":[229],"impact":[230,264],"validity.":[233],"Though":[234],"typically":[235],"small,":[236],"differences":[238],"estimated":[240],"predicted":[241],"were":[243],"sometimes":[244],"large,":[245],"thus":[247,275],"inappropriate":[248],"choices":[249],"lead":[254],"misleading":[256],"estimations":[257],"Such":[263],"did":[265],"normally":[267],"change":[268],"ranking":[270],"between":[271],"models,":[273],"conclusions":[276],"terms":[278],"model":[282],"performs":[283],"better":[284],"likely":[286],"reliable":[287],"independent":[288],"choice":[291],"time,":[294],"especially":[295],"when":[296],"considered":[297],"across":[298],"projects.":[299]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}