{"id":"https://openalex.org/W4401905935","doi":"https://doi.org/10.1109/icst60714.2024.00019","title":"METAL: Metamorphic Testing Framework for Analyzing Large-Language Model Qualities","display_name":"METAL: Metamorphic Testing Framework for Analyzing Large-Language Model Qualities","alternate_titles":[],"publication_year":2024,"publication_date":"2024-05-27","ids":{"openalex":"https://openalex.org/W4401905935","doi":"https://doi.org/10.1109/icst60714.2024.00019"},"language":"en","primary_location":{"id":"doi:10.1109/icst60714.2024.00019","doi":"https://doi.org/10.1109/icst60714.2024.00019","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst60714.2024.00019","pdf_url":null,"source":{"id":"https://openalex.org/S7407085955","display_name":"IEEE International Conference on Software Testing, Verification and Validation (ICST)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W4401905935"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2312.06056","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Sangwon Hyun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I4405259444","https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Sangwon Hyun","raw_affiliation_strings":["School of Computer and Mathematical Sciences, The University of Adelaide,Centre for Research on Engineering Software Technologies (CREST),Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer and Mathematical Sciences, The University of Adelaide,Centre for Research on Engineering Software Technologies (CREST),Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019911052","display_name":"Mingyu Guo","orcid":"https://orcid.org/0000-0002-3478-9201","observed_orcids":["https://orcid.org/0000-0002-3478-9201"]},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I4405259444","https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Mingyu Guo","raw_affiliation_strings":["School of Computer and Mathematical Sciences, The University of Adelaide,Centre for Research on Engineering Software Technologies (CREST),Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer and Mathematical Sciences, The University of Adelaide,Centre for Research on Engineering Software Technologies (CREST),Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103075476","display_name":"Muhammad Ali Babar","orcid":"https://orcid.org/0000-0001-9696-3626","observed_orcids":["https://orcid.org/0000-0001-9696-3626"]},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I4405259444","https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"M. Ali Babar","raw_affiliation_strings":["School of Computer and Mathematical Sciences, The University of Adelaide,Centre for Research on Engineering Software Technologies (CREST),Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer and Mathematical Sciences, The University of Adelaide,Centre for Research on Engineering Software Technologies (CREST),Australia","institution_ids":["https://openalex.org/I5681781"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I5681781"],"apc_list":null,"apc_paid":null,"fwci":5.5205,"has_fulltext":true,"cited_by_count":68,"citation_normalized_percentile":{"value":0.96864354,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"117","last_page":"128"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.8125,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.8125,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.18310000002384186,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metamorphic-testing","display_name":"metamorphic testing","score":0.9660000205039978},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9660000205039978},{"id":"https://openalex.org/keywords/metamorphic-relations","display_name":"metamorphic relations","score":0.9620000123977661},{"id":"https://openalex.org/keywords/attack-success-rate","display_name":"attack success rate","score":0.871999979019165},{"id":"https://openalex.org/keywords/llm-robustness","display_name":"LLM robustness","score":0.7490000128746033},{"id":"https://openalex.org/keywords/automatic-test-generation","display_name":"automated test generation","score":0.3569999933242798}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6486700177192688},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4687167704105377},{"id":"https://openalex.org/C26687426","wikidata":"https://www.wikidata.org/wiki/Q47069","display_name":"Metamorphic rock","level":2,"score":0.4166216552257538},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3661905527114868},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.2818329930305481},{"id":"https://openalex.org/C17409809","wikidata":"https://www.wikidata.org/wiki/Q161764","display_name":"Geochemistry","level":1,"score":0.20426642894744873}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/icst60714.2024.00019","doi":"https://doi.org/10.1109/icst60714.2024.00019","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icst60714.2024.00019","pdf_url":null,"source":{"id":"https://openalex.org/S7407085955","display_name":"IEEE International Conference on Software Testing, Verification and Validation (ICST)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Conference on Software Testing, Verification and Validation (ICST)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2312.06056","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.06056","pdf_url":"https://arxiv.org/pdf/2312.06056","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2312.06056","doi":"https://doi.org/10.48550/arxiv.2312.06056","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2312.06056","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2312.06056","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.06056","pdf_url":"https://arxiv.org/pdf/2312.06056","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401905935.pdf","grobid_xml":"https://content.openalex.org/works/W4401905935.grobid-xml"},"referenced_works_count":54,"referenced_works":["https://openalex.org/W1840435438","https://openalex.org/W2782311202","https://openalex.org/W2794557536","https://openalex.org/W2903495827","https://openalex.org/W2946609015","https://openalex.org/W2950700477","https://openalex.org/W2953084091","https://openalex.org/W2963323070","https://openalex.org/W2963969878","https://openalex.org/W2996851481","https://openalex.org/W3015001695","https://openalex.org/W3017003177","https://openalex.org/W3034850762","https://openalex.org/W3120394327","https://openalex.org/W3128630643","https://openalex.org/W3159933051","https://openalex.org/W3183224878","https://openalex.org/W3185425515","https://openalex.org/W3208206463","https://openalex.org/W3216348338","https://openalex.org/W4213308398","https://openalex.org/W4214720563","https://openalex.org/W4220747294","https://openalex.org/W4221165593","https://openalex.org/W4224215750","https://openalex.org/W4224308101","https://openalex.org/W4229022551","https://openalex.org/W4253021324","https://openalex.org/W4256046779","https://openalex.org/W4283815370","https://openalex.org/W4293302779","https://openalex.org/W4300337452","https://openalex.org/W4309395891","https://openalex.org/W4312570070","https://openalex.org/W4319049323","https://openalex.org/W4320003957","https://openalex.org/W4321854923","https://openalex.org/W4366459745","https://openalex.org/W4378189609","https://openalex.org/W4378465135","https://openalex.org/W4379880468","https://openalex.org/W4380267320","https://openalex.org/W4384302825","https://openalex.org/W4385374425","https://openalex.org/W4385571232","https://openalex.org/W4389518968","https://openalex.org/W4392414346","https://openalex.org/W4399750638","https://openalex.org/W4402460553","https://openalex.org/W4402860127","https://openalex.org/W4404515031","https://openalex.org/W6854112902","https://openalex.org/W6855355790","https://openalex.org/W6973495579"],"related_works":["https://openalex.org/W7161289227","https://openalex.org/W4415746319","https://openalex.org/W7171235819","https://openalex.org/W7125899555","https://openalex.org/W4417339076","https://openalex.org/W7207543244","https://openalex.org/W4410392463","https://openalex.org/W4416250307","https://openalex.org/W4417452048","https://openalex.org/W7125973578"],"abstract_inverted_index":{"Large-Language":[0],"Models":[1],"(LLMs)":[2],"have":[3,34,54,72,179],"shifted":[4],"the":[5,24,84,112,154,161,165,171,182,195,201,207,216],"paradigm":[6],"of":[7,26,44,58,86,115,136,156,168],"natural":[8],"language":[9],"data":[10],"processing.":[11],"However,":[12,51],"their":[13,56,87],"black-boxed":[14],"and":[15,60,64,144,193,214],"probabilistic":[16],"characteristics":[17],"can":[18,132,205],"lead":[19],"to":[20,67,82,99,152],"potential":[21],"risks":[22,197],"in":[23,28,62,198],"quality":[25,196],"outputs":[27],"diverse":[29],"LLM":[30,116,191],"applications.":[31],"Recent":[32],"studies":[33,53,71],"tested":[35],"Quality":[36],"Attributes":[37],"(QAs),":[38],"such":[39],"as":[40,125],"robustness":[41],"or":[42],"fairness,":[43],"LLMs":[45,63,96],"by":[46,103,118,159],"generating":[47,221],"adversarial":[48],"input":[49],"texts.":[50],"existing":[52],"limited":[55],"coverage":[57],"QAs":[59,143,188],"tasks":[61,192],"are":[65],"difficult":[66],"extend.":[68],"Additionally,":[69],"these":[70,101],"only":[73],"used":[74],"one":[75],"evaluation":[76,127],"metric,":[77],"Attack":[78],"Success":[79],"Rate":[80],"(ASR),":[81],"assess":[83,153],"effectiveness":[85,155],"approaches.":[88],"We":[89],"propose":[90],"a":[91],"MEtamorphic":[92],"Testing":[93,106],"for":[94,210,220],"Analyzing":[95],"(METAL)":[97],"framework":[98,131,184],"address":[100],"issues":[102],"applying":[104],"Metamorphic":[105,120],"(MT)":[107],"techniques.":[108],"This":[109],"approach":[110],"facilitates":[111],"systematic":[113],"testing":[114,211],"qualities":[117,167],"defining":[119],"Relations":[121],"(MRs),":[122],"which":[123],"serve":[124],"modularized":[126],"metrics.":[128],"The":[129],"METAL":[130,183],"automatically":[133],"generate":[134],"hundreds":[135],"MRs":[137,157,209],"from":[138],"templates":[139],"that":[140,181],"cover":[141],"various":[142],"tasks.":[145],"In":[146],"addition,":[147],"we":[148,178],"introduced":[149],"novel":[150],"metrics":[151,204],"accurately":[158],"integrating":[160],"ASR":[162],"method":[163,219],"into":[164],"semantic":[166],"text.":[169],"Through":[170],"experiments":[172],"conducted":[173],"with":[174],"three":[175],"prominent":[176],"LLMs,":[177],"confirmed":[180],"effectively":[185],"evaluates":[186],"essential":[187],"on":[189],"primary":[190],"reveals":[194],"LLMs.":[199],"Moreover,":[200],"newly":[202],"proposed":[203],"guide":[206],"optimal":[208],"each":[212],"task":[213],"suggest":[215],"most":[217],"effective":[218],"MRs.":[222]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4401905935","counts_by_year":[{"year":2026,"cited_by_count":39},{"year":2025,"cited_by_count":26},{"year":2024,"cited_by_count":3}],"updated_date":"2026-11-19T14:11:59.513116","created_date":"2024-08-28T00:00:00"}
