{"id":"https://openalex.org/W4252026936","doi":"https://doi.org/10.1109/secse.2013.6615099","title":"Techniques for testing scientific programs without an oracle","display_name":"Techniques for testing scientific programs without an oracle","publication_year":2013,"publication_date":"2013-05-01","ids":{"openalex":"https://openalex.org/W4252026936","doi":"https://doi.org/10.1109/secse.2013.6615099"},"language":"en","primary_location":{"id":"doi:10.1109/secse.2013.6615099","is_oa":false,"landing_page_url":"https://doi.org/10.1109/secse.2013.6615099","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2013 5th International Workshop on Software Engineering for Computational Science and Engineering (SE-CSE)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083120673","display_name":"Upulee Kanewala","orcid":"https://orcid.org/0000-0002-5954-1616"},"institutions":[{"id":"https://openalex.org/I92446798","display_name":"Colorado State University","ror":"https://ror.org/03k1gpj17","country_code":"US","type":"education","lineage":["https://openalex.org/I92446798"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Upulee Kanewala","raw_affiliation_strings":["Computer Science Department, Colorado State University, Fort Collins, CO, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Colorado State University, Fort Collins, CO, USA","institution_ids":["https://openalex.org/I92446798"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5006828223","display_name":"James M. Bieman","orcid":null},"institutions":[{"id":"https://openalex.org/I92446798","display_name":"Colorado State University","ror":"https://ror.org/03k1gpj17","country_code":"US","type":"education","lineage":["https://openalex.org/I92446798"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James M. Bieman","raw_affiliation_strings":["Computer Science Department, Colorado State University, Fort Collins, CO, USA"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Colorado State University, Fort Collins, CO, USA","institution_ids":["https://openalex.org/I92446798"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5083120673"],"corresponding_institution_ids":["https://openalex.org/I92446798"],"apc_list":null,"apc_paid":null,"fwci":1.4581,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.85152057,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"48","last_page":"57"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.8656083941459656},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7758183479309082},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.6151113510131836},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5985351800918579},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.5047954320907593},{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.48335880041122437},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4713180661201477},{"id":"https://openalex.org/keywords/software-testing","display_name":"Software testing","score":0.45275166630744934},{"id":"https://openalex.org/keywords/keyword-driven-testing","display_name":"Keyword-driven testing","score":0.43887069821357727},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.42894667387008667},{"id":"https://openalex.org/keywords/test-case","display_name":"Test case","score":0.41428038477897644},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3939081132411957},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3713921904563904},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.18498724699020386},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.12038296461105347},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08632054924964905}],"concepts":[{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.8656083941459656},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7758183479309082},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.6151113510131836},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5985351800918579},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5047954320907593},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.48335880041122437},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4713180661201477},{"id":"https://openalex.org/C2984328558","wikidata":"https://www.wikidata.org/wiki/Q188522","display_name":"Software testing","level":3,"score":0.45275166630744934},{"id":"https://openalex.org/C169168650","wikidata":"https://www.wikidata.org/wiki/Q1675637","display_name":"Keyword-driven testing","level":5,"score":0.43887069821357727},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.42894667387008667},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.41428038477897644},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3939081132411957},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3713921904563904},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.18498724699020386},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.12038296461105347},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08632054924964905},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/secse.2013.6615099","is_oa":false,"landing_page_url":"https://doi.org/10.1109/secse.2013.6615099","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2013 5th International Workshop on Software Engineering for Computational Science and Engineering (SE-CSE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W197000801","https://openalex.org/W1543987512","https://openalex.org/W1550050122","https://openalex.org/W1963712724","https://openalex.org/W2016339238","https://openalex.org/W2023556458","https://openalex.org/W2041650849","https://openalex.org/W2043998723","https://openalex.org/W2047437983","https://openalex.org/W2047914009","https://openalex.org/W2049183251","https://openalex.org/W2067544274","https://openalex.org/W2091093620","https://openalex.org/W2092221178","https://openalex.org/W2096451472","https://openalex.org/W2097223288","https://openalex.org/W2103318645","https://openalex.org/W2106860490","https://openalex.org/W2110441383","https://openalex.org/W2114178685","https://openalex.org/W2125055259","https://openalex.org/W2126870663","https://openalex.org/W2133990480","https://openalex.org/W2134124281","https://openalex.org/W2137086189","https://openalex.org/W2141933055","https://openalex.org/W2148306883","https://openalex.org/W2161933248","https://openalex.org/W2168565110","https://openalex.org/W2251166296","https://openalex.org/W4247415240","https://openalex.org/W6632354383","https://openalex.org/W6680859568"],"related_works":["https://openalex.org/W2006615808","https://openalex.org/W2142081319","https://openalex.org/W2158303854","https://openalex.org/W2139069137","https://openalex.org/W4250731515","https://openalex.org/W2810804488","https://openalex.org/W2972383756","https://openalex.org/W1498418524","https://openalex.org/W4213182711","https://openalex.org/W2075147138"],"abstract_inverted_index":{"The":[0],"existence":[1],"of":[2,63,117,119,127,139],"an":[3],"oracle":[4],"is":[5,111],"often":[6],"assumed":[7],"in":[8,12,61],"software":[9],"testing.":[10],"But":[11],"many":[13],"situations,":[14],"especially":[15],"for":[16],"scientific":[17,86],"programs,":[18],"oracles":[19,53],"do":[20,89],"not":[21,90],"exist":[22],"or":[23],"they":[24],"are":[25,36],"too":[26],"hard":[27],"to":[28,38,83,113,146,154],"implement.":[29],"This":[30],"paper":[31],"examines":[32],"three":[33,81],"techniques":[34,82,94],"that":[35,88,131],"used":[37],"test":[39,52,85],"programs":[40,87],"without":[41],"oracles:":[42],"(1)":[43,65,137],"Metamorphic":[44],"testing,":[45],"(2)":[46,69,143],"Run-time":[47],"Assertions":[48],"and":[49,71,150],"(3)":[50,72,151],"Developing":[51],"using":[54],"machine":[55,156],"learning.":[56],"We":[57],"examine":[58],"these":[59,80,120],"methods":[60],"terms":[62],"their":[64],"fault":[66,99],"finding":[67,100],"ability,":[68],"automation,":[70],"required":[73,125],"domain":[74,128],"knowledge.":[75,129],"Several":[76],"case":[77],"studies":[78],"apply":[79],"effectively":[84],"have":[91,95],"oracles.":[92,159],"Certain":[93],"reported":[96],"a":[97],"better":[98],"ability":[101],"than":[102],"the":[103,115,124],"others":[104],"when":[105],"testing":[106],"specific":[107],"programs.":[108],"Finally,":[109],"there":[110],"potential":[112],"increase":[114],"level":[116,126],"automation":[118],"techniques,":[121],"thereby":[122],"reducing":[123],"Techniques":[130],"can":[132],"potentially":[133],"be":[134],"automated":[135],"include":[136],"detection":[138],"likely":[140],"metamorphic":[141],"relations,":[142],"static":[144],"analyses":[145,153],"eliminate":[147],"spurious":[148],"invariants":[149],"structural":[152],"develop":[155],"learning":[157],"generated":[158]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":2},{"year":2014,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}