{"id":"https://openalex.org/W4412703970","doi":"https://doi.org/10.1145/3696630.3728559","title":"Using Large Language Models to Support the Workflow of Differential Testing","display_name":"Using Large Language Models to Support the Workflow of Differential Testing","publication_year":2025,"publication_date":"2025-06-23","ids":{"openalex":"https://openalex.org/W4412703970","doi":"https://doi.org/10.1145/3696630.3728559"},"language":"en","primary_location":{"id":"doi:10.1145/3696630.3728559","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3696630.3728559","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3696630.3728559","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093461014","display_name":"Arun Krishna Vajjala","orcid":"https://orcid.org/0009-0002-5593-1438"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Arun Krishna Vajjala","raw_affiliation_strings":["George Mason University, Fairfax, USA"],"affiliations":[{"raw_affiliation_string":"George Mason University, Fairfax, USA","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5095371007","display_name":"Ajay Krishna Vajjala","orcid":null},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ajay Krishna Vajjala","raw_affiliation_strings":["George Mason University, Fairfax, USA"],"affiliations":[{"raw_affiliation_string":"George Mason University, Fairfax, USA","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104178670","display_name":"Carmen Badea","orcid":"https://orcid.org/0009-0005-2249-6371"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carmen Badea","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036278905","display_name":"Christian Bird","orcid":"https://orcid.org/0000-0002-7774-0387"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christian Bird","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jade D'Souza","orcid":"https://orcid.org/0009-0000-0412-931X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jade D'Souza","raw_affiliation_strings":["Microsoft, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026655012","display_name":"Robert DeLine","orcid":"https://orcid.org/0000-0001-8885-8367"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert DeLine","raw_affiliation_strings":["Microsoft, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119096433","display_name":"Mikhail O Demyanyuk","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mikhail O Demyanyuk","raw_affiliation_strings":["Microsoft, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092912239","display_name":"Jason Entenmann","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jason Entenmann","raw_affiliation_strings":["Microsoft, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069657658","display_name":"Nicole Forsgren","orcid":"https://orcid.org/0000-0003-2263-9326"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicole Forsgren","raw_affiliation_strings":["Microsoft Research, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119096432","display_name":"Aliaksandr Hramadski","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aliaksandr Hramadski","raw_affiliation_strings":["Microsoft, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Haris Mohammad","orcid":"https://orcid.org/0009-0007-3596-591X"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haris Mohammad","raw_affiliation_strings":["Microsoft, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Sandeepan Sanyal","orcid":"https://orcid.org/0009-0004-6749-1438"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sandeepan Sanyal","raw_affiliation_strings":["Microsoft, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042187633","display_name":"Oleg Surmachev","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oleg Surmachev","raw_affiliation_strings":["Microsoft, Redmond, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061605447","display_name":"Thomas Zimmermann","orcid":"https://orcid.org/0000-0003-4905-1469"},"institutions":[{"id":"https://openalex.org/I204250578","display_name":"University of California, Irvine","ror":"https://ror.org/04gyf1771","country_code":"US","type":"education","lineage":["https://openalex.org/I204250578"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thomas Zimmermann","raw_affiliation_strings":["University of California, Irvine, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Irvine, USA","institution_ids":["https://openalex.org/I204250578"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":14,"corresponding_author_ids":["https://openalex.org/A5093461014"],"corresponding_institution_ids":["https://openalex.org/I162714631"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.18111675,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"355","last_page":"365"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7576684951782227},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6911560297012329},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4381556808948517},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3821488916873932},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.1598779261112213}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7576684951782227},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6911560297012329},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4381556808948517},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3821488916873932},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.1598779261112213}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3696630.3728559","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3696630.3728559","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3696630.3728559","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3696630.3728559","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W109452506","https://openalex.org/W2038544212","https://openalex.org/W2060291526","https://openalex.org/W2121984759","https://openalex.org/W2170224888","https://openalex.org/W2340482529","https://openalex.org/W2408634597","https://openalex.org/W2725103509","https://openalex.org/W2783275449","https://openalex.org/W2898887908","https://openalex.org/W2912296587","https://openalex.org/W3014370431","https://openalex.org/W3095252018","https://openalex.org/W3096688843","https://openalex.org/W3152154974","https://openalex.org/W4211024062","https://openalex.org/W4221143515","https://openalex.org/W4231904752","https://openalex.org/W4233145949","https://openalex.org/W4233148883","https://openalex.org/W4234609361","https://openalex.org/W4235670907","https://openalex.org/W4236990373","https://openalex.org/W4239249613","https://openalex.org/W4244165259","https://openalex.org/W4246788636","https://openalex.org/W4320068740","https://openalex.org/W4367674071","https://openalex.org/W4389162342","https://openalex.org/W4390481198","https://openalex.org/W4401641057","https://openalex.org/W4413360892","https://openalex.org/W6604335577","https://openalex.org/W6677739885","https://openalex.org/W6683327911","https://openalex.org/W6729291771","https://openalex.org/W6740356784","https://openalex.org/W6789007201","https://openalex.org/W6843590370","https://openalex.org/W7015439612","https://openalex.org/W7066344169"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W1981780420","https://openalex.org/W2182707996","https://openalex.org/W45233828","https://openalex.org/W2964988449","https://openalex.org/W2397952901","https://openalex.org/W2029380707","https://openalex.org/W188202134"],"abstract_inverted_index":{"Many":[0],"software":[1],"development":[2],"teams":[3],"use":[4,109],"differential":[5,94],"testing":[6],"as":[7,36],"a":[8,21,26,30,84,98,137,166],"quality":[9],"gate":[10],"in":[11,23,28,154],"their":[12,90,113,152],"release":[13],"process.":[14],"Differential":[15],"testing\u2014namely,":[16],"comparing":[17],"behavioral":[18],"differences":[19,120],"between":[20],"system":[22,27],"production":[24],"and":[25,62,121,127],"test\u2014is":[29],"laborious":[31],"process":[32,52],"to":[33,46,88,102,108,111,150,162],"label":[34,139],"changes":[35,42],"regressions,":[37],"expected":[38],"changes,":[39],"or":[40,48],"incidental":[41],"(e.g.":[43],"those":[44],"due":[45],"nondeterminism":[47],"timing).":[49],"This":[50],"manual":[51],"involves":[53],"inspecting":[54],"large":[55],"textual":[56],"artifacts,":[57],"like":[58],"logs,":[59],"pull":[60],"requests,":[61],"team":[63,82,133],"discussions,":[64],"which":[65],"suggests":[66],"that":[67,135],"Large":[68],"Language":[69],"Models":[70],"(LLMs)":[71],"could":[72],"be":[73,163],"helpful.":[74],"In":[75],"this":[76],"paper,":[77],"we":[78],"engage":[79],"with":[80],"the":[81,132,155,159],"developing":[83],"central":[85],"Azure":[86],"service":[87],"understand":[89],"work":[91,114],"practice":[92],"for":[93],"testing.":[95],"We":[96],"used":[97],"design":[99],"probe":[100],"method":[101],"elicit":[103],"feedback":[104],"about":[105],"several":[106],"ways":[107],"LLMs":[110],"improve":[112,151],"practice,":[115],"including":[116],"automatically":[117],"labeling":[118],"behavior":[119],"providing":[122],"summaries":[123,161],"of":[124],"various":[125],"artifacts":[126],"discussions.":[128],"Release":[129],"engineers":[130],"on":[131],"report":[134],"predicting":[136],"difference's":[138],"would":[140],"save":[141],"them":[142],"effort,":[143],"but":[144],"they":[145,157],"want":[146],"an":[147],"explicit":[148],"rationale":[149],"trust":[153],"prediction;":[156],"found":[158],"generated":[160],"informative,":[164],"if":[165],"bit":[167],"wordy.":[168]},"counts_by_year":[],"updated_date":"2026-04-17T18:11:37.981687","created_date":"2025-10-10T00:00:00"}