{"id":"https://openalex.org/W4410538875","doi":"https://doi.org/10.1109/icst62969.2025.10989025","title":"LLMs in the Heart of Differential Testing: A Case Study on a Medical Rule Engine","display_name":"LLMs in the Heart of Differential Testing: A Case Study on a Medical Rule Engine","publication_year":2025,"publication_date":"2025-03-31","ids":{"openalex":"https://openalex.org/W4410538875","doi":"https://doi.org/10.1109/icst62969.2025.10989025"},"language":"en","primary_location":{"id":"doi:10.1109/icst62969.2025.10989025","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst62969.2025.10989025","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092991020","display_name":"Erblin Isaku","orcid":null},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]},{"id":"https://openalex.org/I2799829267","display_name":"Simula Research Laboratory","ror":"https://ror.org/00vn06n10","country_code":"NO","type":"facility","lineage":["https://openalex.org/I2799829267"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Erblin Isaku","raw_affiliation_strings":["Simula Research Laboratory and University of Oslo,Oslo,Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Simula Research Laboratory and University of Oslo,Oslo,Norway","institution_ids":["https://openalex.org/I2799829267","https://openalex.org/I184942183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031026274","display_name":"Christoph Laaber","orcid":"https://orcid.org/0000-0001-6817-331X"},"institutions":[{"id":"https://openalex.org/I2799829267","display_name":"Simula Research Laboratory","ror":"https://ror.org/00vn06n10","country_code":"NO","type":"facility","lineage":["https://openalex.org/I2799829267"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Christoph Laaber","raw_affiliation_strings":["Simula Research Laboratory,Oslo,Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Simula Research Laboratory,Oslo,Norway","institution_ids":["https://openalex.org/I2799829267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072908431","display_name":"Hassan Sartaj","orcid":"https://orcid.org/0000-0001-5212-9787"},"institutions":[{"id":"https://openalex.org/I2799829267","display_name":"Simula Research Laboratory","ror":"https://ror.org/00vn06n10","country_code":"NO","type":"facility","lineage":["https://openalex.org/I2799829267"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Hassan Sartaj","raw_affiliation_strings":["Simula Research Laboratory,Oslo,Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Simula Research Laboratory,Oslo,Norway","institution_ids":["https://openalex.org/I2799829267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076142098","display_name":"Shaukat Ali","orcid":"https://orcid.org/0000-0002-9979-3519"},"institutions":[{"id":"https://openalex.org/I2799829267","display_name":"Simula Research Laboratory","ror":"https://ror.org/00vn06n10","country_code":"NO","type":"facility","lineage":["https://openalex.org/I2799829267"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Shaukat Ali","raw_affiliation_strings":["Simula Research Laboratory,Oslo,Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Simula Research Laboratory,Oslo,Norway","institution_ids":["https://openalex.org/I2799829267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114549488","display_name":"Thomas Schwitalla","orcid":"https://orcid.org/0000-0002-0286-1686"},"institutions":[{"id":"https://openalex.org/I4210129856","display_name":"Cancer Registry of Norway","ror":"https://ror.org/03sm1ej59","country_code":"NO","type":"archive","lineage":["https://openalex.org/I1333353642","https://openalex.org/I4210129856","https://openalex.org/I4210149801","https://openalex.org/I4210163359"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Thomas Schwitalla","raw_affiliation_strings":["Cancer Registry of Norway,Oslo,Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cancer Registry of Norway,Oslo,Norway","institution_ids":["https://openalex.org/I4210129856"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017631834","display_name":"Jan F. Nyg\u00e5rd","orcid":"https://orcid.org/0000-0001-9655-7003"},"institutions":[{"id":"https://openalex.org/I4210129856","display_name":"Cancer Registry of Norway","ror":"https://ror.org/03sm1ej59","country_code":"NO","type":"archive","lineage":["https://openalex.org/I1333353642","https://openalex.org/I4210129856","https://openalex.org/I4210149801","https://openalex.org/I4210163359"]},{"id":"https://openalex.org/I78037679","display_name":"UiT The Arctic University of Norway","ror":"https://ror.org/00wge5k78","country_code":"NO","type":"education","lineage":["https://openalex.org/I78037679"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Jan F. Nyg\u00e5rd","raw_affiliation_strings":["Cancer Registry of Norway and UiT The Arctic University of Norway,Oslo,Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cancer Registry of Norway and UiT The Arctic University of Norway,Oslo,Norway","institution_ids":["https://openalex.org/I4210129856","https://openalex.org/I78037679"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.2137,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.94127294,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"429","last_page":"440"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9537000060081482,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9537000060081482,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/differential","display_name":"Differential (mechanical device)","score":0.5194405913352966},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4619484543800354},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.3445412516593933},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.19900363683700562}],"concepts":[{"id":"https://openalex.org/C93226319","wikidata":"https://www.wikidata.org/wiki/Q193137","display_name":"Differential (mechanical device)","level":2,"score":0.5194405913352966},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4619484543800354},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.3445412516593933},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.19900363683700562},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icst62969.2025.10989025","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst62969.2025.10989025","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W1596515083","https://openalex.org/W1986024718","https://openalex.org/W2014538668","https://openalex.org/W2041282815","https://openalex.org/W2041713059","https://openalex.org/W2135841285","https://openalex.org/W2138428785","https://openalex.org/W2154897437","https://openalex.org/W2767556175","https://openalex.org/W2770050759","https://openalex.org/W2782311202","https://openalex.org/W2890801208","https://openalex.org/W2997401484","https://openalex.org/W3043201130","https://openalex.org/W3086938529","https://openalex.org/W4251988601","https://openalex.org/W4254332277","https://openalex.org/W4281763794","https://openalex.org/W4308731473","https://openalex.org/W4323033692","https://openalex.org/W4376606797","https://openalex.org/W4378170222","https://openalex.org/W4378591002","https://openalex.org/W4384009687","https://openalex.org/W4384561707","https://openalex.org/W4384918448","https://openalex.org/W4385570982","https://openalex.org/W4386185444","https://openalex.org/W4386554742","https://openalex.org/W4387500346","https://openalex.org/W4387711873","https://openalex.org/W4388483492","https://openalex.org/W4389544230","https://openalex.org/W6604335577","https://openalex.org/W6682213787","https://openalex.org/W6721965189","https://openalex.org/W6819811686","https://openalex.org/W6852119581","https://openalex.org/W6853945670","https://openalex.org/W6854926761","https://openalex.org/W6855483941","https://openalex.org/W6856556076","https://openalex.org/W6858522248","https://openalex.org/W6859825556","https://openalex.org/W6860167363"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W3031052312","https://openalex.org/W4389568370","https://openalex.org/W3032375762","https://openalex.org/W1995515455","https://openalex.org/W2080531066","https://openalex.org/W3108674512","https://openalex.org/W1506200166"],"abstract_inverted_index":{"The":[0],"Cancer":[1],"Registry":[2],"of":[3,39,79,134],"Norway":[4],"(CRN)":[5],"uses":[6],"an":[7,97],"automated":[8],"cancer":[9,17],"registration":[10],"support":[11,15],"system":[12],"(CaReSS)":[13],"to":[14,89,106,125,137,144],"core":[16,37],"registry":[18],"activities,":[19],"i.e.,":[20],"data":[21,23,27,47],"capture,":[22],"curation,":[24],"and":[25,29,65,101,120,132,140,162,198],"producing":[26],"products":[28],"statistics":[30],"for":[31,44,92,196],"various":[32],"stakeholders.":[33],"GURI":[34],"is":[35,42,158,163],"a":[36,76],"component":[38],"CaReSS,":[40],"which":[41],"responsible":[43],"validating":[45],"incoming":[46],"with":[48,111],"medical":[49,52,58,62,116,123,180],"rules.":[50],"Such":[51],"rules":[53,124,181],"are":[54],"manually":[55],"implemented":[56],"by":[57],"experts":[59],"based":[60,200],"on":[61,75,201],"standards,":[63],"regulations,":[64],"research.":[66],"Since":[67],"large":[68,77],"language":[69],"models":[70],"(LLMs)":[71],"have":[72],"been":[73],"trained":[74],"amount":[78],"public":[80],"information,":[81],"including":[82],"these":[83,141],"documents,":[84],"they":[85],"can":[86],"be":[87],"employed":[88],"generate":[90,138],"tests":[91],"GURI.":[93,108,149],"Thus,":[94],"we":[95,193],"propose":[96],"LLM-based":[98],"test":[99,107],"generation":[100],"differential":[102,176],"testing":[103,177],"approach":[104],"(LLMeDiff)":[105],"We":[109],"experimented":[110],"four":[112],"different":[113],"LLMs,":[114],"two":[115],"rule":[117,190],"engine":[118],"implementations,":[119],"58":[121],"real":[122],"investigate":[126],"the":[127,135,156,159,165,171,202],"hallucination,":[128],"success,":[129],"time":[130,173],"efficiency,":[131],"robustness":[133],"LLMs":[136],"tests,":[139],"tests'":[142],"ability":[143],"find":[145],"potential":[146],"issues":[147],"in":[148],"Our":[150,175],"results":[151],"showed":[152],"that":[153],"GPT-3.5":[154],"hallucinates":[155],"least,":[157],"most":[160,166],"successful,":[161],"generally":[164],"robust;":[167],"however,":[168],"it":[169],"has":[170],"worst":[172],"efficiency.":[174],"revealed":[178],"22":[179],"where":[182],"implementation":[183],"inconsistencies":[184],"were":[185],"discovered":[186],"(e.g.,":[187],"regarding":[188],"handling":[189],"versions).":[191],"Finally,":[192],"provide":[194],"insights":[195],"practitioners":[197],"researchers":[199],"results.":[203]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}