{"id":"https://openalex.org/W4400582805","doi":"https://doi.org/10.1145/3643767","title":"COSTELLO: Contrastive Testing for Embedding-Based Large Language Model as a Service Embeddings","display_name":"COSTELLO: Contrastive Testing for Embedding-Based Large Language Model as a Service Embeddings","publication_year":2024,"publication_date":"2024-07-12","ids":{"openalex":"https://openalex.org/W4400582805","doi":"https://doi.org/10.1145/3643767"},"language":"en","primary_location":{"id":"doi:10.1145/3643767","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3643767","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3643767","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102374363","display_name":"Weipeng Jiang","orcid":"https://orcid.org/0000-0002-0382-6401"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weipeng Jiang","raw_affiliation_strings":["Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0002-0382-6401","affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071575216","display_name":"Juan Zhai","orcid":"https://orcid.org/0000-0001-5017-8016"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juan Zhai","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, USA"],"raw_orcid":"https://orcid.org/0000-0001-5017-8016","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101594068","display_name":"Shiqing Ma","orcid":"https://orcid.org/0000-0003-1551-8948"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shiqing Ma","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, USA"],"raw_orcid":"https://orcid.org/0000-0003-1551-8948","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092424261","display_name":"Xiaoyu Zhang","orcid":"https://orcid.org/0000-0001-7010-6749"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoyu Zhang","raw_affiliation_strings":["Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-7010-6749","affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101843177","display_name":"Chao Shen","orcid":"https://orcid.org/0000-0002-6959-0569"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chao Shen","raw_affiliation_strings":["Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0002-6959-0569","affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.5557,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.70507517,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"1","issue":"FSE","first_page":"906","last_page":"928"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9869999885559082,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9833999872207642,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.6324023008346558},{"id":"https://openalex.org/keywords/costello-syndrome","display_name":"Costello syndrome","score":0.6302120685577393},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5366123914718628},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.4312751889228821},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.4197336733341217},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4006408154964447},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.256225049495697},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.11552110314369202},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.10590672492980957},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.10332107543945312}],"concepts":[{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.6324023008346558},{"id":"https://openalex.org/C2779944767","wikidata":"https://www.wikidata.org/wiki/Q1136492","display_name":"Costello syndrome","level":5,"score":0.6302120685577393},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5366123914718628},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.4312751889228821},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.4197336733341217},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4006408154964447},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.256225049495697},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.11552110314369202},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.10590672492980957},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.10332107543945312},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C2781187634","wikidata":"https://www.wikidata.org/wiki/Q20969980","display_name":"KRAS","level":4,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C501734568","wikidata":"https://www.wikidata.org/wiki/Q42918","display_name":"Mutation","level":3,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3643767","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3643767","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3643767","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3643767","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W2081580037","https://openalex.org/W2614322402","https://openalex.org/W2786411768","https://openalex.org/W2804337238","https://openalex.org/W2892341857","https://openalex.org/W2898868990","https://openalex.org/W2954629067","https://openalex.org/W2963913218","https://openalex.org/W2964882141","https://openalex.org/W2968940383","https://openalex.org/W2970352191","https://openalex.org/W2982138961","https://openalex.org/W3002398329","https://openalex.org/W3034368386","https://openalex.org/W3090423701","https://openalex.org/W3090988182","https://openalex.org/W3091407209","https://openalex.org/W3095319910","https://openalex.org/W3098042509","https://openalex.org/W3100869665","https://openalex.org/W3104663419","https://openalex.org/W3105928338","https://openalex.org/W3108655343","https://openalex.org/W3123045479","https://openalex.org/W3128630643","https://openalex.org/W3138154797","https://openalex.org/W3163644064","https://openalex.org/W3194983542","https://openalex.org/W3200167504","https://openalex.org/W3217463290","https://openalex.org/W4210497109","https://openalex.org/W4224313754","https://openalex.org/W4254003040","https://openalex.org/W4280626533","https://openalex.org/W4284675318","https://openalex.org/W4284681038","https://openalex.org/W4285490440","https://openalex.org/W4285490473","https://openalex.org/W4287891464","https://openalex.org/W4294875919","https://openalex.org/W4313563818","https://openalex.org/W4384302803","https://openalex.org/W4384345795"],"related_works":["https://openalex.org/W4281686043","https://openalex.org/W1989981922","https://openalex.org/W2284539496","https://openalex.org/W4311487220","https://openalex.org/W4282049869","https://openalex.org/W1960846834","https://openalex.org/W2170098154","https://openalex.org/W2162849071","https://openalex.org/W2056493923","https://openalex.org/W1977719041"],"abstract_inverted_index":{"Large":[0],"language":[1,251],"models":[2,252],"have":[3,63,74],"gained":[4],"significant":[5],"popularity":[6],"and":[7,19,39,85,96,152,167,176,194,213,232,253],"are":[8],"often":[9],"provided":[10],"as":[11],"a":[12,43,110,119],"service":[13],"(i.e.,":[14],"LLMaaS).":[15],"Companies":[16],"like":[17,61],"OpenAI":[18],"Google":[20],"provide":[21],"online":[22],"APIs":[23],"of":[24,69,83,98,106,148,165,236,273,283],"LLMs":[25,141],"to":[26,30,122,186,208,238],"allow":[27],"downstream":[28,92,284],"users":[29],"create":[31],"innovative":[32],"applications.":[33,93,285],"Despite":[34],"its":[35],"popularity,":[36],"LLM":[37],"safety":[38],"quality":[40],"assurance":[41],"is":[42,138,185],"well-recognized":[44],"concern":[45],"in":[46,67,126,157,220,278],"the":[47,71,81,103,145,149,158,162,210,221,228],"real":[48],"world,":[49],"requiring":[50],"extra":[51],"efforts":[52],"for":[53],"testing":[54,84,109,134],"these":[55],"LLMs.":[56],"Unfortunately,":[57],"while":[58],"end-to-end":[59],"services":[60],"ChatGPT":[62],"garnered":[64],"rising":[65],"attention":[66],"terms":[68],"testing,":[70],"LLMaaS":[72,131,166],"embeddings":[73,89],"comparatively":[75],"received":[76],"less":[77],"scrutiny.":[78],"We":[79,243],"state":[80],"importance":[82],"uncovering":[86],"problematic":[87,181],"individual":[88],"without":[90],"considering":[91],"The":[94,183],"abstraction":[95],"non-interpretability":[97],"embedded":[99],"vectors,":[100],"combined":[101],"with":[102,179,190,216],"black-box":[104,120],"inaccessibility":[105],"LLMaaS,":[107],"make":[108],"challenging":[111],"puzzle.":[112],"This":[113],"paper":[114],"proposes":[115],"C":[116,245,262],"ostello":[117,246,263],",":[118],"approach":[121],"reveal":[123],"potential":[124,180,240],"defects":[125],"abstract":[127],"embedding":[128,233],"vectors":[129],"from":[130],"by":[132,197],"contrastive":[133,188],".":[135],"Our":[136,201],"intuition":[137],"that":[139,261],"high-quality":[140],"can":[142,171,226,264],"adequately":[143],"capture":[144],"semantic":[146,267],"relationships":[147,156,219],"input":[150],"texts":[151],"properly":[153],"represent":[154],"their":[155],"high-dimensional":[159,222],"space.":[160,223],"For":[161],"given":[163],"interface":[164],"seed":[168,199],"inputs,":[169],"Costello":[170],"automatically":[172],"generate":[173,214],"test":[174],"suites":[175],"output":[177],"words":[178],"embeddings.":[182],"idea":[184],"synthesize":[187],"samples":[189,215],"guidance,":[191],"including":[192],"positive":[193],"negative":[195],"samples,":[196],"mutating":[198],"inputs.":[200],"synthesis":[202],"guide":[203],"will":[204],"leverage":[205],"task-specific":[206],"properties":[207],"control":[209],"mutation":[211],"procedure":[212],"known":[217],"partial":[218],"Thus,":[224],"we":[225],"compare":[227],"expected":[229],"relationship":[230],"(oracle)":[231],"distance":[234],"(output":[235],"LLMs)":[237],"locate":[239],"buggy":[241],"cases.":[242],"evaluate":[244],"on":[247,275],"42":[248],"open-source":[249],"(encoder-based)":[250],"two":[254],"real-world":[255],"commercial":[256],"LLMaaS.":[257],"Experimental":[258],"results":[259],"show":[260],"effectively":[265],"detect":[266],"violations,":[268],"where":[269],"more":[270],"than":[271],"62%":[272],"violations":[274],"average":[276],"result":[277],"erroneous":[279],"behaviors":[280],"(e.g.,":[281],"unfairness)":[282]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}