{"id":"https://openalex.org/W4384030289","doi":"https://doi.org/10.1145/3607184","title":"Testing Causality in Scientific Modelling Software","display_name":"Testing Causality in Scientific Modelling Software","publication_year":2023,"publication_date":"2023-07-12","ids":{"openalex":"https://openalex.org/W4384030289","doi":"https://doi.org/10.1145/3607184"},"language":"en","primary_location":{"id":"doi:10.1145/3607184","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3607184","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059699673","display_name":"Andrew G. Clark","orcid":"https://orcid.org/0000-0002-6830-0566"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Andrew G. Clark","raw_affiliation_strings":["The University of Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102753979","display_name":"Michael S. Foster","orcid":"https://orcid.org/0000-0001-8233-9873"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Michael Foster","raw_affiliation_strings":["The University of Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075364570","display_name":"Benedikt Prifling","orcid":"https://orcid.org/0000-0002-2952-0208"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Benedikt Prifling","raw_affiliation_strings":["Ulm University, Germany"],"affiliations":[{"raw_affiliation_string":"Ulm University, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034625397","display_name":"Neil Walkinshaw","orcid":"https://orcid.org/0000-0003-2134-6548"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Neil Walkinshaw","raw_affiliation_strings":["The University of Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009644797","display_name":"Robert M. Hierons","orcid":"https://orcid.org/0000-0002-4771-1446"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Robert M. Hierons","raw_affiliation_strings":["The University of Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042036645","display_name":"Volker Schmidt","orcid":"https://orcid.org/0000-0001-7451-8185"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Volker Schmidt","raw_affiliation_strings":["Ulm University, Germany"],"affiliations":[{"raw_affiliation_string":"Ulm University, Germany","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027542461","display_name":"Robert D. Turner","orcid":"https://orcid.org/0000-0002-1353-1404"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Robert D. Turner","raw_affiliation_strings":["The University of Sheffield, United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Sheffield, United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5059699673"],"corresponding_institution_ids":["https://openalex.org/I91136226"],"apc_list":null,"apc_paid":null,"fwci":2.806,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.94427752,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"33","issue":"1","first_page":"1","last_page":"42"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7780673503875732},{"id":"https://openalex.org/keywords/causal-inference","display_name":"Causal inference","score":0.7361326217651367},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5543062686920166},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5261216759681702},{"id":"https://openalex.org/keywords/causal-model","display_name":"Causal model","score":0.4881068170070648},{"id":"https://openalex.org/keywords/causality","display_name":"Causality (physics)","score":0.45631536841392517},{"id":"https://openalex.org/keywords/causal-structure","display_name":"Causal structure","score":0.44592466950416565},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.440583735704422},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2953333258628845},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.167592853307724},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11314699053764343}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7780673503875732},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.7361326217651367},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5543062686920166},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5261216759681702},{"id":"https://openalex.org/C11671645","wikidata":"https://www.wikidata.org/wiki/Q5054567","display_name":"Causal model","level":2,"score":0.4881068170070648},{"id":"https://openalex.org/C64357122","wikidata":"https://www.wikidata.org/wiki/Q1149766","display_name":"Causality (physics)","level":2,"score":0.45631536841392517},{"id":"https://openalex.org/C163504300","wikidata":"https://www.wikidata.org/wiki/Q2364925","display_name":"Causal structure","level":2,"score":0.44592466950416565},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.440583735704422},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2953333258628845},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.167592853307724},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11314699053764343},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3607184","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3607184","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2584656196","display_name":null,"funder_award_id":"EP/T030526/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G2897631617","display_name":"CITCoM: Casual Inference for Testing of Computational Models","funder_award_id":"EP/T030526/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":93,"referenced_works":["https://openalex.org/W84018622","https://openalex.org/W436069389","https://openalex.org/W788330839","https://openalex.org/W1480909796","https://openalex.org/W1502922572","https://openalex.org/W1509343160","https://openalex.org/W1516659296","https://openalex.org/W1527688737","https://openalex.org/W1608087281","https://openalex.org/W1647643282","https://openalex.org/W1787500456","https://openalex.org/W1842847600","https://openalex.org/W1909085655","https://openalex.org/W1974075673","https://openalex.org/W1978108654","https://openalex.org/W1979236986","https://openalex.org/W2004929506","https://openalex.org/W2011762419","https://openalex.org/W2015616421","https://openalex.org/W2017425457","https://openalex.org/W2021672791","https://openalex.org/W2026252407","https://openalex.org/W2027946111","https://openalex.org/W2028146834","https://openalex.org/W2030978593","https://openalex.org/W2031874479","https://openalex.org/W2040932457","https://openalex.org/W2041713059","https://openalex.org/W2045671792","https://openalex.org/W2049910836","https://openalex.org/W2056640446","https://openalex.org/W2061439988","https://openalex.org/W2097441841","https://openalex.org/W2110441383","https://openalex.org/W2118166339","https://openalex.org/W2119296494","https://openalex.org/W2124054158","https://openalex.org/W2124621264","https://openalex.org/W2126818664","https://openalex.org/W2130566074","https://openalex.org/W2137819283","https://openalex.org/W2143117649","https://openalex.org/W2145064897","https://openalex.org/W2148306883","https://openalex.org/W2148385623","https://openalex.org/W2152456876","https://openalex.org/W2161482941","https://openalex.org/W2208550830","https://openalex.org/W2324595780","https://openalex.org/W2334860424","https://openalex.org/W2341416175","https://openalex.org/W2343875716","https://openalex.org/W2395419086","https://openalex.org/W2466989778","https://openalex.org/W2735301415","https://openalex.org/W2770263637","https://openalex.org/W2888100319","https://openalex.org/W2889801262","https://openalex.org/W2927351257","https://openalex.org/W2948579453","https://openalex.org/W2963692345","https://openalex.org/W2996483157","https://openalex.org/W3008797115","https://openalex.org/W3027579178","https://openalex.org/W3042943713","https://openalex.org/W3046551854","https://openalex.org/W3081541271","https://openalex.org/W3084123562","https://openalex.org/W3091241784","https://openalex.org/W3095633647","https://openalex.org/W3098649977","https://openalex.org/W3101981467","https://openalex.org/W3105183509","https://openalex.org/W3114319529","https://openalex.org/W3121647782","https://openalex.org/W3124373176","https://openalex.org/W3126633874","https://openalex.org/W3140054544","https://openalex.org/W3172039697","https://openalex.org/W3175026797","https://openalex.org/W3179349298","https://openalex.org/W3186633320","https://openalex.org/W3193505628","https://openalex.org/W3195068169","https://openalex.org/W3196469412","https://openalex.org/W3210311544","https://openalex.org/W4233730851","https://openalex.org/W4236523229","https://openalex.org/W4245581594","https://openalex.org/W4285805284","https://openalex.org/W4294553991","https://openalex.org/W4301715823","https://openalex.org/W4323076409"],"related_works":["https://openalex.org/W2161504683","https://openalex.org/W2574301230","https://openalex.org/W4386620154","https://openalex.org/W4389961576","https://openalex.org/W4294555408","https://openalex.org/W4378718308","https://openalex.org/W4372260129","https://openalex.org/W1153243621","https://openalex.org/W4299782962","https://openalex.org/W4386839885"],"abstract_inverted_index":{"From":[0],"simulating":[1],"galaxy":[2],"formation":[3],"to":[4,54,80,125,134,177],"viral":[5],"transmission":[6],"in":[7,16],"a":[8,13,33,57,84,118,143,149],"pandemic,":[9],"scientific":[10,18,44,159,184],"models":[11,45],"play":[12],"pivotal":[14],"role":[15],"developing":[17],"theories":[19],"and":[20,64,104],"supporting":[21],"government":[22],"policy":[23],"decisions":[24],"that":[25,49,120],"affect":[26],"us":[27],"all.":[28],"Given":[29],"these":[30],"critical":[31],"applications,":[32],"poor":[34],"modelling":[35,185],"assumption":[36],"or":[37],"bug":[38],"could":[39],"have":[40],"far-reaching":[41],"consequences.":[42],"However,":[43],"possess":[46],"several":[47],"properties":[48],"make":[50],"them":[51],"notoriously":[52],"difficult":[53],"test,":[55],"including":[56],"complex":[58],"input":[59],"space,":[60],"long":[61],"execution":[62],"times,":[63],"non-determinism,":[65],"rendering":[66],"existing":[67,130],"testing":[68,116,137,165,183],"techniques":[69,124],"impractical.":[70],"In":[71],"fields":[72],"such":[73,145],"as":[74,88,146],"epidemiology,":[75],"where":[76],"researchers":[77],"seek":[78],"answers":[79],"challenging":[81],"causal":[82,99,115,122,127,164],"questions,":[83],"statistical":[85],"methodology":[86],"known":[87],"Causal":[89],"inference":[90,97,123],"has":[91],"addressed":[92],"similar":[93],"problems,":[94],"enabling":[95,132],"the":[96,114,140,163],"of":[98,108,142],"conclusions":[100],"from":[101,129,172],"noisy,":[102],"biased,":[103],"sparse":[105],"data":[106,176],"instead":[107],"costly":[109],"experiments.":[110],"This":[111],"article":[112],"introduces":[113],"framework:":[117],"framework":[119,166],"uses":[121],"establish":[126],"effects":[128],"data,":[131],"users":[133],"conduct":[135],"software":[136],"activities":[138],"concerning":[139],"effect":[141],"change,":[144],"metamorphic":[147,169],"testing,":[148],"posteriori":[150],".":[151],"We":[152],"present":[153],"three":[154],"case":[155],"studies":[156],"covering":[157],"real-world":[158],"models,":[160],"demonstrating":[161],"how":[162],"can":[167],"infer":[168],"test":[170,175],"outcomes":[171],"reused,":[173],"confounded":[174],"provide":[178],"an":[179],"efficient":[180],"solution":[181],"for":[182],"software.":[186]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":4}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}