{"id":"https://openalex.org/W7203483459","doi":"https://doi.org/10.1109/icosaas68663.2026.11648543","title":"Discovering and Repairing Blind Spots in LLM-as-a-Judge Evaluation of Multi-Turn AI Systems","display_name":"Discovering and Repairing Blind Spots in LLM-as-a-Judge Evaluation of Multi-Turn AI Systems","publication_year":2026,"publication_date":"2026-07-01","ids":{"openalex":"https://openalex.org/W7203483459","doi":"https://doi.org/10.1109/icosaas68663.2026.11648543"},"language":"en","primary_location":{"id":"doi:10.1109/icosaas68663.2026.11648543","doi":"https://doi.org/10.1109/icosaas68663.2026.11648543","is_oa":true,"landing_page_url":"https://doi.org/10.1109/icosaas68663.2026.11648543","pdf_url":null,"source":{"id":"https://openalex.org/S7407096249","display_name":"International Conference on Intelligent and Sustainable AI Systems (ICOSAAS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2026 International Conference on Intelligent and Sustainable AI Systems (ICOSAAS)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1109/icosaas68663.2026.11648543","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5147160086","display_name":"Vinay Gummadavelli","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vinay Gummadavelli","raw_affiliation_strings":["Cisco"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cisco","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136981886","display_name":"Usman Imtiaz","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Usman Imtiaz","raw_affiliation_strings":["Washington University of Science and Technology (WUST)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Washington University of Science and Technology (WUST)","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5143710966","display_name":"Prabagaran A","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Prabagaran A","raw_affiliation_strings":["Arcserve India Software Solutions (P) Limited"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arcserve India Software Solutions (P) Limited","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124733132","display_name":"Siva Hemanth Kolla","orcid":"https://orcid.org/0009-0009-2644-5298","observed_orcids":["https://orcid.org/0009-0009-2644-5298"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Siva Hemanth Kolla","raw_affiliation_strings":["Cloud Bridge Solutions Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cloud Bridge Solutions Inc","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147207194","display_name":"Naga Srinivas Bavirisetty","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210125199","display_name":"WSFS Financial Corporation (United States)","ror":"https://ror.org/03zhv0t98","country_code":"US","type":"company","lineage":["https://openalex.org/I4210125199"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Naga Srinivas Bavirisetty","raw_affiliation_strings":["WSFS Bank"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"WSFS Bank","institution_ids":["https://openalex.org/I4210125199"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5147175641","display_name":"Saanvika Reddy Kondapalli","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Saanvika Reddy Kondapalli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.0758,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.9509178,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1611","last_page":"1618"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.13920000195503235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.13920000195503235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.13130000233650208,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.09009999781847,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.9729999899864197},{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.8330000042915344},{"id":"https://openalex.org/keywords/blind-spots","display_name":"blind spots","score":0.7080000042915344},{"id":"https://openalex.org/keywords/fault-prediction","display_name":"fault prediction","score":0.7149999737739563},{"id":"https://openalex.org/keywords/calibration","display_name":"calibration","score":0.5329999923706055},{"id":"https://openalex.org/keywords/annotation-cost-reduction","display_name":"annotation cost reduction","score":0.800000011920929},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.5529999732971191}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5486000180244446},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.525600016117096},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3382999897003174},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.30070000886917114},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.2962999939918518},{"id":"https://openalex.org/C64731932","wikidata":"https://www.wikidata.org/wiki/Q371090","display_name":"Blind spot","level":2,"score":0.27000001072883606},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.25540000200271606},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.23270000517368317},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.23109999299049377},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.22920000553131104}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icosaas68663.2026.11648543","doi":"https://doi.org/10.1109/icosaas68663.2026.11648543","is_oa":true,"landing_page_url":"https://doi.org/10.1109/icosaas68663.2026.11648543","pdf_url":null,"source":{"id":"https://openalex.org/S7407096249","display_name":"International Conference on Intelligent and Sustainable AI Systems (ICOSAAS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2026 International Conference on Intelligent and Sustainable AI Systems (ICOSAAS)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1109/icosaas68663.2026.11648543","doi":"https://doi.org/10.1109/icosaas68663.2026.11648543","is_oa":true,"landing_page_url":"https://doi.org/10.1109/icosaas68663.2026.11648543","pdf_url":null,"source":{"id":"https://openalex.org/S7407096249","display_name":"International Conference on Intelligent and Sustainable AI Systems (ICOSAAS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2026 International Conference on Intelligent and Sustainable AI Systems (ICOSAAS)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W4380052873","https://openalex.org/W4386071547","https://openalex.org/W4387709867","https://openalex.org/W4390824300","https://openalex.org/W4393452486","https://openalex.org/W4398234485","https://openalex.org/W4401043313","https://openalex.org/W4401819987","https://openalex.org/W4402670748","https://openalex.org/W4402671297","https://openalex.org/W4402671620","https://openalex.org/W4403660437","https://openalex.org/W4406015426","https://openalex.org/W7133246896"],"related_works":["https://openalex.org/W7196978130","https://openalex.org/W4416035115","https://openalex.org/W7203482267","https://openalex.org/W7197001468","https://openalex.org/W7163811477","https://openalex.org/W7138342642","https://openalex.org/W7121321095","https://openalex.org/W7164650696","https://openalex.org/W7166875540","https://openalex.org/W7138303952"],"abstract_inverted_index":{"The":[0,100],"widespread":[1],"use":[2],"of":[3,35,85,98,152],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"as":[8,59,178,181],"automated":[9],"evaluators":[10],"for":[11],"multi-turn":[12],"conversational":[13],"AI":[14],"systems":[15,25,190],"is":[16],"due":[17],"to":[18,32,39,63,89,103,112,131,134],"their":[19],"scalability":[20],"and":[21,54,68,94,116,128,137,154,162],"adaptability.":[22],"Nonetheless,":[23],"LLM-as-a-judge":[24],"often":[26],"have":[27],"systemic":[28],"blind":[29],"spots,":[30],"leading":[31],"the":[33,104,143,150],"neglect":[34],"certain":[36],"flaws":[37],"because":[38],"insufficient,":[40],"inflexible,":[41],"or":[42],"biased":[43],"assessment":[44,189],"standards.":[45],"This":[46],"project":[47],"investigates":[48],"if":[49],"disparities":[50],"between":[51],"human":[52,132,171],"annotations":[53],"LLM":[55],"evaluations":[56],"might":[57,186],"function":[58],"a":[60,74,78,120,182],"diagnostic":[61],"instrument":[62],"identify":[64,83],"missing":[65],"evaluation":[66],"criteria":[67],"improve":[69],"rubric":[70,105],"design.":[71],"By":[72],"developing":[73],"rubric,":[75],"we":[76],"provide":[77,119],"systematic":[79],"framework":[80],"that":[81,124,142,170,185],"may":[82],"indications":[84],"disagreement,":[86],"categorize":[87],"them":[88,130],"uncover":[90],"concealed":[91],"defect":[92,147,157],"patterns,":[93],"initiate":[95],"new":[96],"aspects":[97],"evaluation.":[99],"proposed":[101,144],"improvements":[102],"are":[106],"assessed":[107],"using":[108],"previously":[109],"inspected":[110],"datasets":[111],"verify":[113],"its":[114],"universality":[115],"reliability.":[117],"We":[118],"confidence-aware":[121],"abstention":[122],"mechanism":[123],"detects":[125],"doubtful":[126],"predictions":[127],"refers":[129],"assessors":[133],"enhance":[135,187],"safety":[136],"dependability.":[138],"Experimental":[139],"results":[140],"demonstrate":[141],"technique":[145],"improves":[146],"memory,":[148],"optimizes":[149],"calibration":[151],"confidence":[153],"accuracy":[155],"in":[156],"prediction,":[158],"decreases":[159],"false":[160],"alarms,":[161],"lowers":[163],"total":[164],"annotation":[165],"costs.":[166],"Our":[167],"findings":[168],"suggest":[169],"disagreement":[172],"should":[173],"be":[174],"regarded":[175],"not":[176],"just":[177],"noise,":[179],"but":[180],"crucial":[183],"signal":[184],"LLM-based":[188],"over":[191],"time.":[192]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-08-15T00:00:00"}