{"id":"https://openalex.org/W2041713059","doi":"https://doi.org/10.1109/tse.2014.2372785","title":"The Oracle Problem in Software Testing: A Survey","display_name":"The Oracle Problem in Software Testing: A Survey","publication_year":2014,"publication_date":"2014-11-20","ids":{"openalex":"https://openalex.org/W2041713059","doi":"https://doi.org/10.1109/tse.2014.2372785","mag":"2041713059"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2014.2372785","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2014.2372785","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/7106034/06963470.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ieeexplore.ieee.org/ielx7/32/7106034/06963470.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076587279","display_name":"Earl T. Barr","orcid":"https://orcid.org/0000-0003-0771-7891"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Earl T. Barr","raw_affiliation_strings":["Department of Computer Science, University College London, Gower Street, London WC2R 2LS, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University College London, Gower Street, London WC2R 2LS, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000019783","display_name":"Mark Harman","orcid":"https://orcid.org/0000-0002-5864-4488"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mark Harman","raw_affiliation_strings":["Department of Computer Science, University College London, Gower Street, London WC2R 2LS, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University College London, Gower Street, London WC2R 2LS, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078598020","display_name":"Phil McMinn","orcid":"https://orcid.org/0000-0001-9137-7433"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Phil McMinn","raw_affiliation_strings":["Department of Computer Science, University of Sheffield, Sheffield S1 4DP, South Yorkshire, United Kingdom","[Department of Computer Science, University of Sheffield, Sheffield S1 4DP, South Yorkshire, United Kingdom]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Sheffield, Sheffield S1 4DP, South Yorkshire, United Kingdom","institution_ids":["https://openalex.org/I91136226"]},{"raw_affiliation_string":"[Department of Computer Science, University of Sheffield, Sheffield S1 4DP, South Yorkshire, United Kingdom]","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082536208","display_name":"Muzammil Shahbaz","orcid":"https://orcid.org/0000-0002-2183-662X"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Muzammil Shahbaz","raw_affiliation_strings":["Department of Computer Science, University of Sheffield, Sheffield S1 4DP, South Yorkshire, United Kingdom","[Department of Computer Science, University of Sheffield, Sheffield S1 4DP, South Yorkshire, United Kingdom]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Sheffield, Sheffield S1 4DP, South Yorkshire, United Kingdom","institution_ids":["https://openalex.org/I91136226"]},{"raw_affiliation_string":"[Department of Computer Science, University of Sheffield, Sheffield S1 4DP, South Yorkshire, United Kingdom]","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070401254","display_name":"Shin Yoo","orcid":"https://orcid.org/0000-0002-0836-6993"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Shin Yoo","raw_affiliation_strings":["Department of Computer Science, University College London, Gower Street, London WC2R 2LS, London, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University College London, Gower Street, London WC2R 2LS, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":{"value":1256,"currency":"EUR","value_usd":1354},"fwci":88.9121,"has_fulltext":true,"cited_by_count":1068,"citation_normalized_percentile":{"value":1.0,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"41","issue":"5","first_page":"507","last_page":"525"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8393303155899048},{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.72817462682724},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.7081336975097656},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.5576942563056946},{"id":"https://openalex.org/keywords/software-performance-testing","display_name":"Software performance testing","score":0.46619313955307007},{"id":"https://openalex.org/keywords/software-reliability-testing","display_name":"Software reliability testing","score":0.46439129114151},{"id":"https://openalex.org/keywords/regression-testing","display_name":"Regression testing","score":0.4373294711112976},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4336458146572113},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3855117857456207},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.383091002702713},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.29124853014945984}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8393303155899048},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.72817462682724},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.7081336975097656},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5576942563056946},{"id":"https://openalex.org/C178059732","wikidata":"https://www.wikidata.org/wiki/Q1982529","display_name":"Software performance testing","level":5,"score":0.46619313955307007},{"id":"https://openalex.org/C52928878","wikidata":"https://www.wikidata.org/wiki/Q7554226","display_name":"Software reliability testing","level":5,"score":0.46439129114151},{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.4373294711112976},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4336458146572113},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3855117857456207},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.383091002702713},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.29124853014945984}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tse.2014.2372785","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2014.2372785","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/7106034/06963470.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:eprints.ucl.ac.uk.OAI2:1471263","is_oa":false,"landing_page_url":"https://discovery.ucl.ac.uk/id/eprint/1471263/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400024","display_name":"UCL Discovery (University College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45129253","host_organization_name":"University College London","host_organization_lineage":["https://openalex.org/I45129253"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE TRANSACTIONS ON SOFTWARE ENGINEERING , 41 (5) pp. 507-525. (2015)","raw_type":"Article"}],"best_oa_location":{"id":"doi:10.1109/tse.2014.2372785","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2014.2372785","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/7106034/06963470.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2759119072","display_name":"RE-COST: REducing the Cost of Oracles for Software Testing","funder_award_id":"EP/I010165/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G3264601377","display_name":"RE-COST: REducing the Cost of Oracles for Software Testing","funder_award_id":"EP/I010386/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G3986165987","display_name":null,"funder_award_id":"EP/I010165/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G5515407205","display_name":null,"funder_award_id":"EP/I010386/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320320286","display_name":"University College London","ror":"https://ror.org/02jx3x895"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"},{"id":"https://openalex.org/F4320338075","display_name":"Core Research for Evolutional Science and Technology","ror":"https://ror.org/00097mb19"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2041713059.pdf","grobid_xml":"https://content.openalex.org/works/W2041713059.grobid-xml"},"referenced_works_count":270,"referenced_works":["https://openalex.org/W11348788","https://openalex.org/W56856764","https://openalex.org/W61720063","https://openalex.org/W82953342","https://openalex.org/W83755378","https://openalex.org/W97502223","https://openalex.org/W116656696","https://openalex.org/W169243799","https://openalex.org/W1485458514","https://openalex.org/W1492527352","https://openalex.org/W1493173186","https://openalex.org/W1498342204","https://openalex.org/W1498946538","https://openalex.org/W1502951647","https://openalex.org/W1519148184","https://openalex.org/W1520252399","https://openalex.org/W1520408295","https://openalex.org/W1521494223","https://openalex.org/W1536010170","https://openalex.org/W1538137835","https://openalex.org/W1548432010","https://openalex.org/W1552507696","https://openalex.org/W1554638243","https://openalex.org/W1567915323","https://openalex.org/W1572969462","https://openalex.org/W1573007082","https://openalex.org/W1576972802","https://openalex.org/W1576979152","https://openalex.org/W1583295953","https://openalex.org/W1590429295","https://openalex.org/W1596127723","https://openalex.org/W1597210183","https://openalex.org/W1604756141","https://openalex.org/W1607166662","https://openalex.org/W1608087281","https://openalex.org/W1679977702","https://openalex.org/W1702158037","https://openalex.org/W1723067587","https://openalex.org/W1756881852","https://openalex.org/W1760635954","https://openalex.org/W1828540440","https://openalex.org/W1856380672","https://openalex.org/W1866918939","https://openalex.org/W1884646655","https://openalex.org/W1885303164","https://openalex.org/W1895387792","https://openalex.org/W1926460446","https://openalex.org/W1953313694","https://openalex.org/W1963712724","https://openalex.org/W1964446022","https://openalex.org/W1966242430","https://openalex.org/W1966995059","https://openalex.org/W1973326708","https://openalex.org/W1974091547","https://openalex.org/W1975043576","https://openalex.org/W1975934503","https://openalex.org/W1978429203","https://openalex.org/W1989445634","https://openalex.org/W1990543981","https://openalex.org/W1992570401","https://openalex.org/W1992987499","https://openalex.org/W1993760289","https://openalex.org/W1994321811","https://openalex.org/W1994350081","https://openalex.org/W1996062855","https://openalex.org/W1996574389","https://openalex.org/W1998099342","https://openalex.org/W1998393968","https://openalex.org/W2002934700","https://openalex.org/W2004617458","https://openalex.org/W2004929506","https://openalex.org/W2005150099","https://openalex.org/W2005453221","https://openalex.org/W2009477740","https://openalex.org/W2009489720","https://openalex.org/W2011367000","https://openalex.org/W2011715836","https://openalex.org/W2014515160","https://openalex.org/W2015310615","https://openalex.org/W2015999522","https://openalex.org/W2021418918","https://openalex.org/W2022151497","https://openalex.org/W2023740829","https://openalex.org/W2024254950","https://openalex.org/W2029117638","https://openalex.org/W2032680490","https://openalex.org/W2034874525","https://openalex.org/W2035892768","https://openalex.org/W2037291277","https://openalex.org/W2039568841","https://openalex.org/W2041474397","https://openalex.org/W2043709926","https://openalex.org/W2044427446","https://openalex.org/W2045090661","https://openalex.org/W2045671792","https://openalex.org/W2051940067","https://openalex.org/W2055267228","https://openalex.org/W2060561921","https://openalex.org/W2065705928","https://openalex.org/W2069300761","https://openalex.org/W2072083761","https://openalex.org/W2072357626","https://openalex.org/W2074273322","https://openalex.org/W2081814494","https://openalex.org/W2082434633","https://openalex.org/W2082700856","https://openalex.org/W2083411793","https://openalex.org/W2084901307","https://openalex.org/W2088802044","https://openalex.org/W2092221178","https://openalex.org/W2092234295","https://openalex.org/W2095974519","https://openalex.org/W2096204075","https://openalex.org/W2096449544","https://openalex.org/W2097223288","https://openalex.org/W2098800875","https://openalex.org/W2099444174","https://openalex.org/W2099529102","https://openalex.org/W2099855131","https://openalex.org/W2100230555","https://openalex.org/W2100414549","https://openalex.org/W2101512909","https://openalex.org/W2102519055","https://openalex.org/W2102932947","https://openalex.org/W2104000753","https://openalex.org/W2104169671","https://openalex.org/W2106860490","https://openalex.org/W2107356891","https://openalex.org/W2107729695","https://openalex.org/W2107751709","https://openalex.org/W2109276114","https://openalex.org/W2110337982","https://openalex.org/W2110441383","https://openalex.org/W2110908283","https://openalex.org/W2111654822","https://openalex.org/W2112640524","https://openalex.org/W2113944334","https://openalex.org/W2114178685","https://openalex.org/W2114695104","https://openalex.org/W2114869486","https://openalex.org/W2115105266","https://openalex.org/W2115309705","https://openalex.org/W2115484804","https://openalex.org/W2117866489","https://openalex.org/W2119283374","https://openalex.org/W2119296494","https://openalex.org/W2119582290","https://openalex.org/W2122186426","https://openalex.org/W2122987706","https://openalex.org/W2123863902","https://openalex.org/W2123864913","https://openalex.org/W2124081952","https://openalex.org/W2124621264","https://openalex.org/W2125831829","https://openalex.org/W2126495607","https://openalex.org/W2129304537","https://openalex.org/W2129360963","https://openalex.org/W2129942570","https://openalex.org/W2130560480","https://openalex.org/W2132478304","https://openalex.org/W2136189469","https://openalex.org/W2136568800","https://openalex.org/W2136656815","https://openalex.org/W2137086189","https://openalex.org/W2137860663","https://openalex.org/W2138369269","https://openalex.org/W2143101641","https://openalex.org/W2144395955","https://openalex.org/W2144973245","https://openalex.org/W2145303987","https://openalex.org/W2146177878","https://openalex.org/W2147826933","https://openalex.org/W2148306883","https://openalex.org/W2148418519","https://openalex.org/W2151223430","https://openalex.org/W2151226386","https://openalex.org/W2151674625","https://openalex.org/W2153285764","https://openalex.org/W2153511411","https://openalex.org/W2154155067","https://openalex.org/W2155233265","https://openalex.org/W2156063539","https://openalex.org/W2156122544","https://openalex.org/W2157309873","https://openalex.org/W2158699648","https://openalex.org/W2162247081","https://openalex.org/W2162661340","https://openalex.org/W2163362445","https://openalex.org/W2163931728","https://openalex.org/W2164827396","https://openalex.org/W2165751179","https://openalex.org/W2166140339","https://openalex.org/W2166486797","https://openalex.org/W2166956498","https://openalex.org/W2168565110","https://openalex.org/W2170108788","https://openalex.org/W2170224888","https://openalex.org/W2170486381","https://openalex.org/W2171207016","https://openalex.org/W2213891260","https://openalex.org/W2378259225","https://openalex.org/W2498671470","https://openalex.org/W2987907651","https://openalex.org/W3033024692","https://openalex.org/W3106729728","https://openalex.org/W3114516458","https://openalex.org/W3148345308","https://openalex.org/W4210635123","https://openalex.org/W4230919050","https://openalex.org/W4232057538","https://openalex.org/W4237492309","https://openalex.org/W4241889940","https://openalex.org/W4242377092","https://openalex.org/W4244800399","https://openalex.org/W4247954915","https://openalex.org/W4251674089","https://openalex.org/W4251912342","https://openalex.org/W4285719527","https://openalex.org/W4361785283","https://openalex.org/W6602313012","https://openalex.org/W6603422705","https://openalex.org/W6603898897","https://openalex.org/W6606810876","https://openalex.org/W6628852112","https://openalex.org/W6629112484","https://openalex.org/W6629841916","https://openalex.org/W6630065119","https://openalex.org/W6630998367","https://openalex.org/W6632070334","https://openalex.org/W6634360926","https://openalex.org/W6634485668","https://openalex.org/W6636099742","https://openalex.org/W6637409347","https://openalex.org/W6637703538","https://openalex.org/W6638092024","https://openalex.org/W6638612105","https://openalex.org/W6639309987","https://openalex.org/W6639359655","https://openalex.org/W6643685448","https://openalex.org/W6644173552","https://openalex.org/W6648666904","https://openalex.org/W6652979367","https://openalex.org/W6655742219","https://openalex.org/W6656146417","https://openalex.org/W6659827164","https://openalex.org/W6660882777","https://openalex.org/W6661863152","https://openalex.org/W6668610532","https://openalex.org/W6668636469","https://openalex.org/W6673547942","https://openalex.org/W6675305031","https://openalex.org/W6676269210","https://openalex.org/W6676883669","https://openalex.org/W6677050620","https://openalex.org/W6677433537","https://openalex.org/W6678048727","https://openalex.org/W6678691818","https://openalex.org/W6682076387","https://openalex.org/W6682333199","https://openalex.org/W6682613750","https://openalex.org/W6682929155","https://openalex.org/W6682965233","https://openalex.org/W6683002196","https://openalex.org/W6683558667","https://openalex.org/W6683601778","https://openalex.org/W6684722589","https://openalex.org/W6688266807","https://openalex.org/W6824137238","https://openalex.org/W7036008149","https://openalex.org/W7048989014"],"related_works":["https://openalex.org/W2362944210","https://openalex.org/W3133844515","https://openalex.org/W1707101138","https://openalex.org/W2541762924","https://openalex.org/W2187840912","https://openalex.org/W2353261385","https://openalex.org/W2939789152","https://openalex.org/W2376559135","https://openalex.org/W1977030523","https://openalex.org/W2103961571"],"abstract_inverted_index":{"Testing":[0],"involves":[1],"examining":[2],"the":[3,20,24,35,59,95,103,129,150],"behaviour":[4,28,66],"of":[5,22,90,98,109,125,134,146,157,163],"a":[6,18,46,143],"system":[7],"in":[8,159],"order":[9],"to":[10,44,62,149],"discover":[11],"potential":[12],"faults.":[13],"Given":[14],"an":[15,155],"input":[16],"for":[17,77],"system,":[19],"challenge":[21],"distinguishing":[23],"corresponding":[25],"desired,":[26],"correct":[27],"from":[29],"potentially":[30],"incorrect":[31],"behavior":[32],"is":[33,42,67,92],"called":[34],"\u201ctest":[36],"oracle":[37,40,57,78,100,121,152],"problem\u201d.":[38],"Test":[39],"automation":[41],"important":[43,161],"remove":[45],"current":[47,147],"bottleneck":[48],"that":[49,118],"inhibits":[50],"greater":[51],"overall":[52],"test":[53,56,72,99,126,151],"automation.":[54],"Without":[55],"automation,":[58,79],"human":[60],"has":[61,74],"determine":[63],"whether":[64],"observed":[65],"correct.":[68],"The":[69],"literature":[70],"on":[71],"oracles":[73],"introduced":[75],"techniques":[76],"including":[80],"modelling,":[81],"specifications,":[82,111],"contract-driven":[83],"development":[84],"and":[85,114,137,154,167],"metamorphic":[86],"testing.":[87],"When":[88],"none":[89],"these":[91],"completely":[93],"adequate,":[94],"final":[96],"source":[97],"information":[101,117],"remains":[102],"human,":[104,131],"who":[105],"may":[106],"be":[107],"aware":[108],"informal":[110,120],"expectations,":[112],"norms":[113],"domain":[115],"specific":[116],"provide":[119],"guidance.":[122],"All":[123],"forms":[124],"oracles,":[127],"even":[128],"humble":[130],"involve":[132],"challenges":[133],"reducing":[135],"cost":[136],"increasing":[138],"benefit.":[139],"This":[140],"paper":[141],"provides":[142],"comprehensive":[144],"survey":[145],"approaches":[148],"problem":[153],"analysis":[156],"trends":[158],"this":[160],"area":[162],"software":[164],"testing":[165],"research":[166],"practice.":[168]},"counts_by_year":[{"year":2026,"cited_by_count":64},{"year":2025,"cited_by_count":128},{"year":2024,"cited_by_count":118},{"year":2023,"cited_by_count":103},{"year":2022,"cited_by_count":105},{"year":2021,"cited_by_count":122},{"year":2020,"cited_by_count":107},{"year":2019,"cited_by_count":90},{"year":2018,"cited_by_count":98},{"year":2017,"cited_by_count":57},{"year":2016,"cited_by_count":59},{"year":2015,"cited_by_count":17}],"updated_date":"2026-08-14T07:06:38.338062","created_date":"2025-10-10T00:00:00"}