{"id":"https://openalex.org/W4400978763","doi":"https://doi.org/10.1109/tse.2024.3433463","title":"Assessing Evaluation Metrics for Neural Test Oracle Generation","display_name":"Assessing Evaluation Metrics for Neural Test Oracle Generation","publication_year":2024,"publication_date":"2024-07-25","ids":{"openalex":"https://openalex.org/W4400978763","doi":"https://doi.org/10.1109/tse.2024.3433463"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2024.3433463","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3433463","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102446495","display_name":"Jiho Shin","orcid":"https://orcid.org/0000-0001-8829-3773"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jiho Shin","raw_affiliation_strings":["Lassonde School of Engineering, York University, Toronto, Canada","Department of EECS York University Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0001-8829-3773","affiliations":[{"raw_affiliation_string":"Lassonde School of Engineering, York University, Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Department of EECS York University Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070634219","display_name":"Hadi Hemmati","orcid":"https://orcid.org/0000-0003-0204-9812"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Hadi Hemmati","raw_affiliation_strings":["Lassonde School of Engineering, York University, Toronto, Canada","Department of EECS York University Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0204-9812","affiliations":[{"raw_affiliation_string":"Lassonde School of Engineering, York University, Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Department of EECS York University Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045306475","display_name":"Moshi Wei","orcid":"https://orcid.org/0000-0003-1659-1960"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Moshi Wei","raw_affiliation_strings":["Lassonde School of Engineering, York University, Toronto, Canada","Department of EECS York University Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1659-1960","affiliations":[{"raw_affiliation_string":"Lassonde School of Engineering, York University, Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Department of EECS York University Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100326213","display_name":"Song Wang","orcid":"https://orcid.org/0000-0003-0531-5717"},"institutions":[{"id":"https://openalex.org/I192455969","display_name":"York University","ror":"https://ror.org/05fq50484","country_code":"CA","type":"education","lineage":["https://openalex.org/I192455969"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Song Wang","raw_affiliation_strings":["Lassonde School of Engineering, York University, Toronto, Canada","Department of EECS York University Toronto, Canada"],"raw_orcid":"https://orcid.org/0000-0003-0531-5717","affiliations":[{"raw_affiliation_string":"Lassonde School of Engineering, York University, Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]},{"raw_affiliation_string":"Department of EECS York University Toronto, Canada","institution_ids":["https://openalex.org/I192455969"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.0181,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.9255737,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"50","issue":"9","first_page":"2337","last_page":"2349"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8667210936546326},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.6919939517974854},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5294684171676636},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4984734058380127},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39667922258377075},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38908851146698},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3756590485572815},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3619140684604645}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8667210936546326},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.6919939517974854},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5294684171676636},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4984734058380127},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39667922258377075},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38908851146698},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3756590485572815},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3619140684604645},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2024.3433463","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2024.3433463","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320325651","display_name":"Alberta Innovates","ror":null},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":65,"referenced_works":["https://openalex.org/W1965194038","https://openalex.org/W1971650562","https://openalex.org/W1989171126","https://openalex.org/W2025914262","https://openalex.org/W2076796892","https://openalex.org/W2088271853","https://openalex.org/W2088802044","https://openalex.org/W2107667898","https://openalex.org/W2107751709","https://openalex.org/W2110591140","https://openalex.org/W2121669067","https://openalex.org/W2292220633","https://openalex.org/W2294699749","https://openalex.org/W2311532076","https://openalex.org/W2327620174","https://openalex.org/W2466388701","https://openalex.org/W2511950321","https://openalex.org/W2610485112","https://openalex.org/W2619271281","https://openalex.org/W2793350103","https://openalex.org/W2950599942","https://openalex.org/W3005628256","https://openalex.org/W3033397011","https://openalex.org/W3035408261","https://openalex.org/W3086938529","https://openalex.org/W3089307846","https://openalex.org/W3098605233","https://openalex.org/W3104907093","https://openalex.org/W3105903381","https://openalex.org/W3108032709","https://openalex.org/W3126675481","https://openalex.org/W3160700180","https://openalex.org/W3161997752","https://openalex.org/W3195727321","https://openalex.org/W4220852596","https://openalex.org/W4284681867","https://openalex.org/W4284690374","https://openalex.org/W4285204876","https://openalex.org/W4287593187","https://openalex.org/W4287668913","https://openalex.org/W4308643033","https://openalex.org/W4312934144","https://openalex.org/W4365452590","https://openalex.org/W4375859932","https://openalex.org/W4378676759","https://openalex.org/W4384155677","https://openalex.org/W4384304865","https://openalex.org/W4384345664","https://openalex.org/W4384345748","https://openalex.org/W4385570337","https://openalex.org/W4387725359","https://openalex.org/W4391979620","https://openalex.org/W4392904204","https://openalex.org/W4393147129","https://openalex.org/W6678262379","https://openalex.org/W6682631176","https://openalex.org/W6752037958","https://openalex.org/W6783958930","https://openalex.org/W6785167183","https://openalex.org/W6790588633","https://openalex.org/W6795855765","https://openalex.org/W6852417946","https://openalex.org/W6858066316","https://openalex.org/W6898505805","https://openalex.org/W6944578371"],"related_works":["https://openalex.org/W2073713056","https://openalex.org/W3110702597","https://openalex.org/W2078761926","https://openalex.org/W2110441383","https://openalex.org/W2125620709","https://openalex.org/W1498872724","https://openalex.org/W4233149903","https://openalex.org/W4293864700","https://openalex.org/W2524540579","https://openalex.org/W2326878701"],"abstract_inverted_index":{"Recently,":[0],"deep":[1,315],"learning":[2,316],"models":[3,16,115],"have":[4,224,266],"shown":[5],"promising":[6],"results":[7],"in":[8,98,282,318,322],"test":[9,57,66,102,112,122,153,186,219,246,261,293,305,320],"oracle":[10,54,82,113,294],"generation.":[11],"Neural":[12],"Oracle":[13],"Generation":[14],"(NOG)":[15],"are":[17,25],"commonly":[18],"evaluated":[19],"using":[20],"static":[21],"(automatic)":[22],"metrics":[23,44,68,124,151,176,188,216,221,258,263,307],"which":[24,59],"mainly":[26],"based":[27],"on":[28,116,161,171,292,301],"textual":[29,42,99,118,149,174,214,256,302],"similarity":[30,43,100,119,150,175,215,257,303],"of":[31,51,95,139,314],"the":[32,48,52,92,148,162,168,178,184,192,203,232,239,245,250,280,283,323],"output,":[33],"e.g.":[34],"BLEU,":[35],"ROUGE-L,":[36],"METEOR,":[37],"and":[38,73,101,108,120,152,304,308],"Accuracy.":[39],"However,":[40,181],"these":[41,135],"may":[45],"not":[46],"reflect":[47],"testing":[49],"effectiveness":[50],"generated":[53],"within":[55,231],"a":[56,198,273],"suite,":[58],"is":[60],"often":[61],"measured":[62],"by":[63],"dynamic":[64],"(execution-based)":[65],"adequacy":[67,103,123,154,187,220,247,262,306],"such":[69],"as":[70],"code":[71],"coverage":[72],"mutation":[74],"score.":[75],"In":[76],"this":[77,287],"work,":[78],"we":[79,106,142],"revisit":[80],"existing":[81],"generation":[83,114,295,321],"studies":[84,291],"plus":[85],"<italic":[86,158,163],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[87,159,164],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">gpt-3.5</i>":[88,160],"to":[89,190,201,223,241,265,267,279],"empirically":[90],"investigate":[91],"current":[93],"standing":[94],"their":[96],"performance":[97,170,299],"metrics.":[104,140,155,248],"Specifically,":[105],"train":[107],"run":[109],"four":[110],"state-of-the-art":[111],"seven":[117,173],"two":[121,130,136],"for":[125,238,311],"our":[126,206],"analysis.":[127],"We":[128,195,208],"apply":[129],"different":[131,137,269,274],"correlation":[132,146],"analyses":[133],"between":[134,147],"sets":[138],"Surprisingly,":[141],"found":[143,209],"no":[144],"significant":[145],"For":[156],"instance,":[157],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">jackrabbit-oak</i>":[165],"project":[166],"had":[167,183],"highest":[169],"all":[172,191],"among":[177],"studied":[179,193],"NOGs.":[180,194],"it":[182],"lowest":[185],"compared":[189],"further":[196],"conducted":[197],"qualitative":[199],"analysis":[200],"explore":[202],"reasons":[204],"behind":[205],"observations.":[207],"that":[210,276],"oracles":[211,253],"with":[212,254,296],"high":[213,260],"but":[217,259],"low":[218,255],"tend":[222,264],"complex":[225],"or":[226,272],"multiple":[227],"chained":[228],"method":[229,275],"invocations":[230],"oracle's":[233],"parameters,":[234],"making":[235],"them":[236],"hard":[237],"model":[240],"generate":[242],"completely,":[243],"affecting":[244],"On":[249],"other":[251],"hand,":[252],"call":[268],"assertion":[270],"types":[271],"functions":[277],"similarly":[278],"ones":[281],"ground":[284],"truth.":[285],"Overall,":[286],"work":[288],"complements":[289],"prior":[290],"an":[297],"extensive":[298],"evaluation":[300],"provides":[309],"guidelines":[310],"better":[312],"assessment":[313],"applications":[317],"software":[319],"future.":[324]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}