{"id":"https://openalex.org/W4406461922","doi":"https://doi.org/10.1109/bigdata62323.2024.10825592","title":"Examining Trustworthiness of LLM-as-a-Judge Systems in a Clinical Trial Design Benchmark","display_name":"Examining Trustworthiness of LLM-as-a-Judge Systems in a Clinical Trial Design Benchmark","publication_year":2024,"publication_date":"2024-12-15","ids":{"openalex":"https://openalex.org/W4406461922","doi":"https://doi.org/10.1109/bigdata62323.2024.10825592"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata62323.2024.10825592","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825592","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076535717","display_name":"Corey Curran","orcid":"https://orcid.org/0009-0003-4799-476X"},"institutions":[{"id":"https://openalex.org/I165799507","display_name":"Rensselaer Polytechnic Institute","ror":"https://ror.org/01rtyzb94","country_code":"US","type":"education","lineage":["https://openalex.org/I165799507"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Corey Curran","raw_affiliation_strings":["Rensselaer Polytechnic Institute,Department of Mathematical Sciences,Troy,New York,12180-3590"],"affiliations":[{"raw_affiliation_string":"Rensselaer Polytechnic Institute,Department of Mathematical Sciences,Troy,New York,12180-3590","institution_ids":["https://openalex.org/I165799507"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010838436","display_name":"Nafis Neehal","orcid":null},"institutions":[{"id":"https://openalex.org/I165799507","display_name":"Rensselaer Polytechnic Institute","ror":"https://ror.org/01rtyzb94","country_code":"US","type":"education","lineage":["https://openalex.org/I165799507"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nafis Neehal","raw_affiliation_strings":["Rensselaer Polytechnic Institute,Department of Computer Science,Troy,New York,12180-3590"],"affiliations":[{"raw_affiliation_string":"Rensselaer Polytechnic Institute,Department of Computer Science,Troy,New York,12180-3590","institution_ids":["https://openalex.org/I165799507"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104188415","display_name":"Keerthiram Murugesan","orcid":null},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Keerthiram Murugesan","raw_affiliation_strings":["IBM Research,NY,10598"],"affiliations":[{"raw_affiliation_string":"IBM Research,NY,10598","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048876983","display_name":"Kristin P. Bennett","orcid":"https://orcid.org/0000-0002-8782-105X"},"institutions":[{"id":"https://openalex.org/I165799507","display_name":"Rensselaer Polytechnic Institute","ror":"https://ror.org/01rtyzb94","country_code":"US","type":"education","lineage":["https://openalex.org/I165799507"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kristin P. Bennett","raw_affiliation_strings":["Rensselaer Polytechnic Institute,Department of Mathematical Sciences,Troy,New York,12180-3590"],"affiliations":[{"raw_affiliation_string":"Rensselaer Polytechnic Institute,Department of Mathematical Sciences,Troy,New York,12180-3590","institution_ids":["https://openalex.org/I165799507"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5076535717"],"corresponding_institution_ids":["https://openalex.org/I165799507"],"apc_list":null,"apc_paid":null,"fwci":2.135,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.92147196,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"4627","last_page":"4631"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10804","display_name":"Health Systems, Economic Evaluations, Quality of Life","score":0.9812999963760376,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10804","display_name":"Health Systems, Economic Evaluations, Quality of Life","score":0.9812999963760376,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10582","display_name":"Ethics in Clinical Research","score":0.9782999753952026,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9736999869346619,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7452964782714844},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.6601715087890625},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6198886632919312},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3413451313972473},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.21515822410583496}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7452964782714844},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.6601715087890625},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6198886632919312},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3413451313972473},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.21515822410583496},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata62323.2024.10825592","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825592","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W49002582","https://openalex.org/W138612281","https://openalex.org/W2061623382","https://openalex.org/W2328330440","https://openalex.org/W2768689913","https://openalex.org/W3193158708","https://openalex.org/W3207678513","https://openalex.org/W4309183852","https://openalex.org/W4384071683","https://openalex.org/W4385373826","https://openalex.org/W4385574097","https://openalex.org/W4389519583","https://openalex.org/W4400104334","https://openalex.org/W4403285227"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4246352526"],"abstract_inverted_index":{"Manual":[0],"evaluation":[1,27,164],"of":[2,25,134],"Large":[3],"Language":[4],"Model":[5],"(LLM)":[6],"applications":[7,168],"at":[8],"scale":[9],"presents":[10],"significant":[11],"resource":[12],"challenges,":[13],"making":[14],"LLM-as-Judge":[15],"(LaaJ)":[16],"an":[17],"attractive":[18],"alternative.":[19],"This":[20],"study":[21],"examines":[22],"the":[23,158],"reliability":[24],"LaaJ":[26,153,176],"within":[28],"CT-Bench,":[29],"a":[30,119,142,171],"benchmark":[31],"for":[32,58,99,102,174],"assessing":[33],"LLMs\u2019":[34],"capabilities":[35],"in":[36,115,160,166],"recommending":[37],"clinical":[38,55],"trial":[39],"baseline":[40,87],"features.":[41],"LaaJ-alpha,":[42],"our":[43],"GPT-4o":[44,103],"based":[45],"prototype,":[46],"semantically":[47],"matches":[48],"LLM-recommended":[49],"features":[50,53,74],"against":[51],"reference":[52],"from":[54,75],"trials,":[56],"accounting":[57],"semantic":[59],"equivalence":[60],"(e.g.,":[61],"\u2018BMI\u2019":[62],"and":[63,72,82,95,104,108,169],"\u2018Body":[64],"Mass":[65],"Index\u2019).":[66],"The":[67,138],"system":[68],"generates":[69],"matched":[70],"pairs":[71],"unmatched":[73],"both":[76],"sources":[77],"to":[78,151],"calculate":[79],"precision,":[80],"recall,":[81],"F1":[83],"scores.":[84],"Laaj-alpha":[85],"evaluates":[86],"feature":[88],"recommendations":[89],"across":[90],"CTBench":[91],"CT-Pub":[92],"(100":[93],"trials)":[94,98],"CT-Repo":[96],"(1,690":[97],"comparing":[100],"results":[101],"Llama-3-70B-Instruct":[105],"under":[106],"zero-shot":[107],"three-shot":[109],"settings.":[110],"Coherence":[111],"checking":[112],"revealed":[113],"hallucinations":[114],"LaaJ-alpha\u2019s":[116],"evaluation,":[117],"necessitating":[118],"post-processing":[120],"correction":[121],"step":[122],"that":[123,146],"yielded":[124],"lower":[125],"but":[126],"more":[127],"accurate":[128],"performance":[129],"metrics.":[130],"Three":[131],"different":[132],"types":[133],"hallucination":[135,139],"were":[136],"observed.":[137],"rate":[140],"provides":[141],"quantifiable":[143],"coherence":[144],"metric":[145],"can":[147],"be":[148],"systematically":[149],"used":[150],"improve":[152],"reliability.":[154],"Our":[155],"findings":[156],"underscore":[157],"challenges":[159],"developing":[161],"reliable":[162],"LLM":[163],"methods":[165],"healthcare":[167],"demonstrate":[170],"potential":[172],"framework":[173],"improving":[175],"systems.":[177]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}