{"id":"https://openalex.org/W2530824252","doi":"https://doi.org/10.1109/tse.2016.2616306","title":"A Framework for Evaluating the Results of the SZZ Approach for Identifying Bug-Introducing Changes","display_name":"A Framework for Evaluating the Results of the SZZ Approach for Identifying Bug-Introducing Changes","publication_year":2016,"publication_date":"2016-10-11","ids":{"openalex":"https://openalex.org/W2530824252","doi":"https://doi.org/10.1109/tse.2016.2616306","mag":"2530824252"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2016.2616306","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2016.2616306","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052196896","display_name":"Daniel Alencar da Costa","orcid":"https://orcid.org/0000-0003-4525-3266"},"institutions":[{"id":"https://openalex.org/I35046152","display_name":"Universidade Federal do Rio Grande do Norte","ror":"https://ror.org/04wn09761","country_code":"BR","type":"education","lineage":["https://openalex.org/I35046152"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Daniel Alencar da Costa","raw_affiliation_strings":["Department of Informatics and Applied Mathematics (DIMAp), Federal University of Rio Grande do Norte, Natal-RN, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Informatics and Applied Mathematics (DIMAp), Federal University of Rio Grande do Norte, Natal-RN, Brazil","institution_ids":["https://openalex.org/I35046152"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043053174","display_name":"Shane McIntosh","orcid":"https://orcid.org/0000-0002-0193-3975"},"institutions":[{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Shane McIntosh","raw_affiliation_strings":["Department of Electrical and Computer Engineering, McGill University, Montreal, QC, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, McGill University, Montreal, QC, Canada","institution_ids":["https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056378414","display_name":"Weiyi Shang","orcid":"https://orcid.org/0000-0001-6222-7444"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Weiyi Shang","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Concordia University, Montreal, QC, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Concordia University, Montreal, QC, Canada","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064571846","display_name":"Uir\u00e1 Kulesza","orcid":"https://orcid.org/0000-0002-5467-6458"},"institutions":[{"id":"https://openalex.org/I35046152","display_name":"Universidade Federal do Rio Grande do Norte","ror":"https://ror.org/04wn09761","country_code":"BR","type":"education","lineage":["https://openalex.org/I35046152"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Uira Kulesza","raw_affiliation_strings":["Department of Informatics and Applied Mathematics (DIMAp), Federal University of Rio Grande do Norte, Natal-RN, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Informatics and Applied Mathematics (DIMAp), Federal University of Rio Grande do Norte, Natal-RN, Brazil","institution_ids":["https://openalex.org/I35046152"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065908206","display_name":"Roberta Coelho","orcid":"https://orcid.org/0000-0003-0001-435X"},"institutions":[{"id":"https://openalex.org/I35046152","display_name":"Universidade Federal do Rio Grande do Norte","ror":"https://ror.org/04wn09761","country_code":"BR","type":"education","lineage":["https://openalex.org/I35046152"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Roberta Coelho","raw_affiliation_strings":["Department of Informatics and Applied Mathematics (DIMAp), Federal University of Rio Grande do Norte, Natal-RN, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Informatics and Applied Mathematics (DIMAp), Federal University of Rio Grande do Norte, Natal-RN, Brazil","institution_ids":["https://openalex.org/I35046152"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091586373","display_name":"Ahmed E. Hassan","orcid":"https://orcid.org/0000-0001-7749-5513"},"institutions":[{"id":"https://openalex.org/I204722609","display_name":"Queen's University","ror":"https://ror.org/02y72wh86","country_code":"CA","type":"education","lineage":["https://openalex.org/I204722609"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Ahmed E. Hassan","raw_affiliation_strings":["Software Analysis and Intelligence Lab (SAIL), Queen\u2019s University, Kingston, ON, Canada","Software Analysis and Intelligence Lab (SAIL), Queen's University, Kingston, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Software Analysis and Intelligence Lab (SAIL), Queen\u2019s University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]},{"raw_affiliation_string":"Software Analysis and Intelligence Lab (SAIL), Queen's University, Kingston, ON, Canada","institution_ids":["https://openalex.org/I204722609"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":41.7808,"has_fulltext":false,"cited_by_count":192,"citation_normalized_percentile":{"value":0.99697984,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"43","issue":"7","first_page":"641","last_page":"657"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.8983685970306396},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7967955470085144},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5337207317352295},{"id":"https://openalex.org/keywords/software-bug","display_name":"Software bug","score":0.527427077293396},{"id":"https://openalex.org/keywords/software-implementation","display_name":"Software implementation","score":0.46147724986076355},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.41010066866874695},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3996409475803375},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.26293110847473145},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.11148872971534729}],"concepts":[{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.8983685970306396},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7967955470085144},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5337207317352295},{"id":"https://openalex.org/C1009929","wikidata":"https://www.wikidata.org/wiki/Q179550","display_name":"Software bug","level":3,"score":0.527427077293396},{"id":"https://openalex.org/C2983609787","wikidata":"https://www.wikidata.org/wiki/Q10534782","display_name":"Software implementation","level":3,"score":0.46147724986076355},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.41010066866874695},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3996409475803375},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.26293110847473145},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.11148872971534729},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2016.2616306","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2016.2616306","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.4399999976158142}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":62,"referenced_works":["https://openalex.org/W78170808","https://openalex.org/W409344806","https://openalex.org/W1966394892","https://openalex.org/W1968745662","https://openalex.org/W1971070436","https://openalex.org/W1973650376","https://openalex.org/W1978813754","https://openalex.org/W1984769753","https://openalex.org/W1987855178","https://openalex.org/W1994248747","https://openalex.org/W2007705030","https://openalex.org/W2024920205","https://openalex.org/W2027483357","https://openalex.org/W2040975718","https://openalex.org/W2041432408","https://openalex.org/W2044396529","https://openalex.org/W2044508485","https://openalex.org/W2049576151","https://openalex.org/W2050496630","https://openalex.org/W2051978688","https://openalex.org/W2061426850","https://openalex.org/W2065314038","https://openalex.org/W2080066761","https://openalex.org/W2098157675","https://openalex.org/W2099858487","https://openalex.org/W2100945416","https://openalex.org/W2103640219","https://openalex.org/W2110385988","https://openalex.org/W2110498832","https://openalex.org/W2110653426","https://openalex.org/W2112708109","https://openalex.org/W2118949539","https://openalex.org/W2126166995","https://openalex.org/W2128666163","https://openalex.org/W2134091519","https://openalex.org/W2147105902","https://openalex.org/W2147386665","https://openalex.org/W2148357053","https://openalex.org/W2149321161","https://openalex.org/W2150733339","https://openalex.org/W2150786161","https://openalex.org/W2150800057","https://openalex.org/W2150874999","https://openalex.org/W2151553346","https://openalex.org/W2152703345","https://openalex.org/W2157353183","https://openalex.org/W2158744032","https://openalex.org/W2164519300","https://openalex.org/W2167117640","https://openalex.org/W2171733741","https://openalex.org/W2405501105","https://openalex.org/W3023328472","https://openalex.org/W3144363571","https://openalex.org/W4237979974","https://openalex.org/W4241947741","https://openalex.org/W4247032699","https://openalex.org/W4256027430","https://openalex.org/W4285719527","https://openalex.org/W6603234767","https://openalex.org/W6674712254","https://openalex.org/W6676396137","https://openalex.org/W6713595552"],"related_works":["https://openalex.org/W2741776607","https://openalex.org/W62555423","https://openalex.org/W150283939","https://openalex.org/W3124070457","https://openalex.org/W2953316694","https://openalex.org/W2380651582","https://openalex.org/W1662636095","https://openalex.org/W623781850","https://openalex.org/W125619821","https://openalex.org/W1872937274"],"abstract_inverted_index":{"The":[0,72],"approach":[1],"proposed":[2,99,117,191],"by":[3,165,204],"Silwerski,":[4],"Zimmermann,":[5],"and":[6,89],"Zeller":[7],"(SZZ)":[8],"for":[9,142,197],"identifying":[10],"bug-introducing":[11,129,137,166,188],"changes":[12,167],"is":[13,44,52,202],"at":[14,156],"the":[15,22,27,49,66,75,79,84,91,98,124,161,199],"foundation":[16],"of":[17,30,68,87,93,126,145,160],"several":[18],"research":[19],"areas":[20],"within":[21],"software":[23],"engineering":[24],"discipline.":[25],"Despite":[26],"foundational":[28],"role":[29],"SZZ,":[31],"little":[32],"effort":[33],"has":[34],"been":[35],"made":[36],"to":[37,64,101,119,122,185],"evaluate":[38,65,102],"its":[39],"results.":[40],"Such":[41,175],"an":[42],"evaluation":[43],"a":[45,62,135,194,205],"challenging":[46],"task":[47],"because":[48],"ground":[50],"truth":[51],"not":[53],"readily":[54],"available.":[55],"By":[56],"acknowledging":[57],"such":[58],"challenges,":[59],"we":[60,149],"propose":[61],"framework":[63,73,100,192],"results":[67,176],"alternative":[69],"SZZ":[70,104,120,152,180,207],"implementations.":[71],"evaluates":[74],"following":[76],"criteria:":[77],"(1)":[78],"earliest":[80],"bug":[81,94],"appearance,":[82],"(2)":[83],"future":[85,146],"impact":[86],"changes,":[88],"(3)":[90],"realism":[92],"introduction.":[95],"We":[96,113,131],"use":[97],"five":[103],"implementations":[105,153,181],"using":[106],"data":[107,200],"from":[108,172],"ten":[109],"open":[110],"source":[111],"projects.":[112],"find":[114,133,150],"that":[115,134,151,155,168,178,201],"previously":[116],"improvements":[118],"tend":[121],"inflate":[123],"number":[125],"incorrectly":[127],"identified":[128],"changes.":[130,189],"also":[132],"single":[136],"change":[138],"may":[139],"be":[140],"blamed":[141],"introducing":[143],"hundreds":[144],"bugs.":[147],"Furthermore,":[148],"report":[154],"least":[157],"46":[158],"percent":[159],"bugs":[162],"are":[163,169],"caused":[164],"years":[170],"apart":[171],"one":[173],"another.":[174],"suggest":[177],"current":[179],"still":[182],"lack":[183],"mechanisms":[184],"accurately":[186],"identify":[187],"Our":[190],"provides":[193],"systematic":[195],"mean":[196],"evaluating":[198],"generated":[203],"given":[206],"implementation.":[208]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":19},{"year":2024,"cited_by_count":28},{"year":2023,"cited_by_count":22},{"year":2022,"cited_by_count":28},{"year":2021,"cited_by_count":26},{"year":2020,"cited_by_count":19},{"year":2019,"cited_by_count":23},{"year":2018,"cited_by_count":19},{"year":2017,"cited_by_count":4}],"updated_date":"2026-07-21T08:15:58.654021","created_date":"2025-10-10T00:00:00"}