{"id":"https://openalex.org/W4400978763","doi":"https://doi.org/10.1109/tse.2024.3433463","title":"Assessing Evaluation Metrics for Neural Test Oracle Generation","display_name":"Assessing Evaluation Metrics for Neural Test Oracle Generation","publication_year":2024,"publication_date":"2024-07-25","ids":{"openalex":"https://openalex.org/W4400978763","doi":"https://doi.org/10.1109/tse.2024.3433463"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3433463","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3433463","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102446495","display_name":"Jiho Shin","orcid":"https://orcid.org/0000-0001-8829-3773"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jiho Shin","raw_affiliation_strings":["Lassonde School of Engineering, York University, Toronto, Canada","Department of EECS York University Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0001-8829-3773","affiliations":[{"raw_affiliation_string":"Lassonde School of Engineering, York University, Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Department of EECS York University Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070634219","display_name":"Hadi Hemmati","orcid":"https://orcid.org/0000-0003-0204-9812"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Hadi Hemmati","raw_affiliation_strings":["Lassonde School of Engineering, York University, Toronto, Canada","Department of EECS York University Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0204-9812","affiliations":[{"raw_affiliation_string":"Lassonde School of Engineering, York University, Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Department of EECS York University Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045306475","display_name":"Moshi Wei","orcid":"https://orcid.org/0000-0003-1659-1960"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Moshi Wei","raw_affiliation_strings":["Lassonde School of Engineering, York University, Toronto, Canada","Department of EECS York University Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1659-1960","affiliations":[{"raw_affiliation_string":"Lassonde School of Engineering, York University, Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Department of EECS York University Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100326213","display_name":"Song Wang","orcid":"https://orcid.org/0000-0003-0531-5717"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Song Wang","raw_affiliation_strings":["Lassonde School of Engineering, York University, Toronto, Canada","Department of EECS York University Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0531-5717","affiliations":[{"raw_affiliation_string":"Lassonde School of Engineering, York University, Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Department of EECS York University Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I192455969"],"apc_list":null,"apc_paid":null,"fwci":3.6695,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.94005149,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"50","issue":"9","first_page":"2337","last_page":"2349"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8667210936546326},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.6919939517974854},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5294684171676636},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4984734058380127},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39667922258377075},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38908851146698},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3756590485572815},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3619140684604645}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8667210936546326},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.6919939517974854},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5294684171676636},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4984734058380127},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39667922258377075},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38908851146698},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3756590485572815},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3619140684604645},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2024.3433463","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3433463","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320325651","display_name":"Alberta Innovates","ror":null},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":65,"referenced_works":["https://openalex.org/W1965194038","https://openalex.org/W1971650562","https://openalex.org/W1989171126","https://openalex.org/W2025914262","https://openalex.org/W2076796892","https://openalex.org/W2088271853","https://openalex.org/W2088802044","https://openalex.org/W2107667898","https://openalex.org/W2107751709","https://openalex.org/W2110591140","https://openalex.org/W2121669067","https://openalex.org/W2292220633","https://openalex.org/W2294699749","https://openalex.org/W2311532076","https://openalex.org/W2327620174","https://openalex.org/W2466388701","https://openalex.org/W2511950321","https://openalex.org/W2610485112","https://openalex.org/W2619271281","https://openalex.org/W2793350103","https://openalex.org/W2950599942","https://openalex.org/W3005628256","https://openalex.org/W3033397011","https://openalex.org/W3035408261","https://openalex.org/W3086938529","https://openalex.org/W3089307846","https://openalex.org/W3098605233","https://openalex.org/W3104907093","https://openalex.org/W3105903381","https://openalex.org/W3108032709","https://openalex.org/W3126675481","https://openalex.org/W3160700180","https://openalex.org/W3161997752","https://openalex.org/W3195727321","https://openalex.org/W4220852596","https://openalex.org/W4284681867","https://openalex.org/W4284690374","https://openalex.org/W4285204876","https://openalex.org/W4287593187","https://openalex.org/W4287668913","https://openalex.org/W4308643033","https://openalex.org/W4312934144","https://openalex.org/W4365452590","https://openalex.org/W4375859932","https://openalex.org/W4378676759","https://openalex.org/W4384155677","https://openalex.org/W4384304865","https://openalex.org/W4384345664","https://openalex.org/W4384345748","https://openalex.org/W4385570337","https://openalex.org/W4387725359","https://openalex.org/W4391979620","https://openalex.org/W4392904204","https://openalex.org/W4393147129","https://openalex.org/W6678262379","https://openalex.org/W6682631176","https://openalex.org/W6752037958","https://openalex.org/W6783958930","https://openalex.org/W6785167183","https://openalex.org/W6790588633","https://openalex.org/W6795855765","https://openalex.org/W6852417946","https://openalex.org/W6858066316","https://openalex.org/W6898505805","https://openalex.org/W6944578371"],"related_works":["https://openalex.org/W2073713056","https://openalex.org/W3110702597","https://openalex.org/W2078761926","https://openalex.org/W2110441383","https://openalex.org/W2125620709","https://openalex.org/W1498872724","https://openalex.org/W4233149903","https://openalex.org/W4293864700","https://openalex.org/W2524540579","https://openalex.org/W2326878701"],"abstract_inverted_index":{"Recently,":[0],"deep":[1,303],"learning":[2,304],"models":[3,16,111],"have":[4,212,254],"shown":[5],"promising":[6],"results":[7],"in":[8,94,270,306,310],"test":[9,57,66,98,108,118,149,174,207,234,249,281,293,308],"oracle":[10,54,82,109,282],"generation.":[11],"Neural":[12],"Oracle":[13],"Generation":[14],"(NOG)":[15],"are":[17,25],"commonly":[18],"evaluated":[19],"using":[20],"static":[21],"(automatic)":[22],"metrics":[23,44,68,120,147,164,176,204,209,246,251,295],"which":[24,59],"mainly":[26],"based":[27],"on":[28,112,159,280,289],"textual":[29,42,95,114,145,162,202,244,290],"similarity":[30,43,96,115,146,163,203,245,291],"of":[31,51,91,135,302],"the":[32,48,52,88,144,156,166,172,180,191,220,227,233,238,268,271,311],"output,":[33],"e.g.":[34],"BLEU,":[35],"ROUGE-L,":[36],"METEOR,":[37],"and":[38,73,97,104,116,148,292,296],"Accuracy.":[39],"However,":[40,169],"these":[41,131],"may":[45],"not":[46],"reflect":[47],"testing":[49],"effectiveness":[50],"generated":[53],"within":[55,219],"a":[56,186,261],"suite,":[58],"is":[60],"often":[61],"measured":[62],"by":[63],"dynamic":[64],"(execution-based)":[65],"adequacy":[67,99,119,150,175,208,235,250,294],"such":[69],"as":[70],"code":[71],"coverage":[72],"mutation":[74],"score.":[75],"In":[76],"this":[77,275],"work,":[78],"we":[79,102,138],"revisit":[80],"existing":[81],"generation":[83,110,283,309],"studies":[84,279],"plusgpt-3.5to":[85],"empirically":[86],"investigate":[87],"current":[89],"standing":[90],"their":[92],"performance":[93,158,287],"metrics.":[100,136,151,236],"Specifically,":[101],"train":[103],"run":[105],"four":[106],"state-of-the-art":[107],"seven":[113,161],"two":[117,126,132],"for":[121,226,299],"our":[122,194],"analysis.":[123],"We":[124,183,196],"apply":[125],"different":[127,133,257,262],"correlation":[128,142],"analyses":[129],"between":[130,143],"sets":[134],"Surprisingly,":[137],"found":[139,197],"no":[140],"significant":[141],"For":[152],"instance,gpt-3.5on":[153],"thejackrabbit-oakproject":[154],"had":[155,171],"highest":[157],"all":[160,179],"among":[165],"studied":[167,181],"NOGs.":[168,182],"it":[170],"lowest":[173],"compared":[177],"to":[178,189,211,229,253,255,267],"further":[184],"conducted":[185],"qualitative":[187],"analysis":[188],"explore":[190],"reasons":[192],"behind":[193],"observations.":[195],"that":[198,264],"oracles":[199,241],"with":[200,242,284],"high":[201,248],"but":[205,247],"low":[206,243],"tend":[210,252],"complex":[213],"or":[214,260],"multiple":[215],"chained":[216],"method":[217,263],"invocations":[218],"oracle's":[221],"parameters,":[222],"making":[223],"them":[224],"hard":[225],"model":[228],"generate":[230],"completely,":[231],"affecting":[232],"On":[237],"other":[239],"hand,":[240],"call":[256],"assertion":[258],"types":[259],"functions":[265],"similarly":[266],"ones":[269],"ground":[272],"truth.":[273],"Overall,":[274],"work":[276],"complements":[277],"prior":[278],"an":[285],"extensive":[286],"evaluation":[288],"provides":[297],"guidelines":[298],"better":[300],"assessment":[301],"applications":[305],"software":[307],"future.":[312]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}