{"id":"https://openalex.org/W7162400249","doi":"https://doi.org/10.48550/arxiv.2605.25440","title":"A Multi-Agent LLM Framework for Rating the Quality of Surgical Feedback","display_name":"A Multi-Agent LLM Framework for Rating the Quality of Surgical Feedback","publication_year":2026,"publication_date":"2026-05-25","ids":{"openalex":"https://openalex.org/W7162400249","doi":"https://doi.org/10.48550/arxiv.2605.25440"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.25440","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25440","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.25440","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136514225","display_name":"Rafal Kocielnik","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kocielnik, Rafal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137012875","display_name":"J. Everett Knudsen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Knudsen, J. Everett","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137028041","display_name":"Steven Y. Cen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cen, Steven Y.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137006783","display_name":"Jasmine Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Jasmine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042002882","display_name":"Cherine H. Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Cherine H.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113193161","display_name":"Atharva Deo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deo, Atharva","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080102308","display_name":"Ujjwal Pasupulety","orcid":"https://orcid.org/0000-0001-8760-2829"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pasupulety, Ujjwal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115569447","display_name":"Peter Wager","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wager, Peter","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137069542","display_name":"Anima Anandkumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anandkumar, Anima","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137004457","display_name":"Andrew J. Hung","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hung, Andrew J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":10,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10916","display_name":"Surgical Simulation and Training","score":0.7961999773979187,"subfield":{"id":"https://openalex.org/subfields/2746","display_name":"Surgery"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T10916","display_name":"Surgical Simulation and Training","score":0.7961999773979187,"subfield":{"id":"https://openalex.org/subfields/2746","display_name":"Surgery"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.036400001496076584,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11894","display_name":"Radiology practices and education","score":0.01940000057220459,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/formative-assessment","display_name":"Formative assessment","score":0.8619999885559082},{"id":"https://openalex.org/keywords/trainer","display_name":"Trainer","score":0.8240000009536743},{"id":"https://openalex.org/keywords/clarity","display_name":"CLARITY","score":0.692300021648407},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6258000135421753},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.598800003528595},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5694000124931335},{"id":"https://openalex.org/keywords/summative-assessment","display_name":"Summative assessment","score":0.3758000135421753},{"id":"https://openalex.org/keywords/peer-feedback","display_name":"Peer feedback","score":0.2957000136375427}],"concepts":[{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.8619999885559082},{"id":"https://openalex.org/C2780463512","wikidata":"https://www.wikidata.org/wiki/Q15122700","display_name":"Trainer","level":2,"score":0.8240000009536743},{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.692300021648407},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6258000135421753},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.598800003528595},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5694000124931335},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4512999951839447},{"id":"https://openalex.org/C152747807","wikidata":"https://www.wikidata.org/wiki/Q1854913","display_name":"Summative assessment","level":3,"score":0.3758000135421753},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.31029999256134033},{"id":"https://openalex.org/C42058189","wikidata":"https://www.wikidata.org/wiki/Q7160400","display_name":"Peer feedback","level":2,"score":0.2957000136375427},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.2953999936580658},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.2856000065803528},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.28380000591278076},{"id":"https://openalex.org/C71405471","wikidata":"https://www.wikidata.org/wiki/Q757012","display_name":"Quality management","level":3,"score":0.2815999984741211},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.27950000762939453},{"id":"https://openalex.org/C3018587665","wikidata":"https://www.wikidata.org/wiki/Q7268696","display_name":"Qualitative analysis","level":3,"score":0.27559998631477356},{"id":"https://openalex.org/C22156102","wikidata":"https://www.wikidata.org/wiki/Q5341294","display_name":"Educational measurement","level":3,"score":0.265500009059906},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.26019999384880066},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.2515999972820282},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.25440","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25440","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.25440","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25440","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7856492400169373,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Verbal":[0],"feedback":[1,26,43,66,99,143,152,165],"delivered":[2],"by":[3,50],"attending":[4],"surgeons":[5],"in":[6,15,30,103,163,184],"the":[7,22,62,104,185],"operating":[8,186],"room":[9,187],"plays":[10],"a":[11,38,92,121,190],"critical":[12],"formative":[13],"role":[14],"resident":[16],"trainee":[17,32,169],"skill":[18],"acquisition.":[19],"Yet,":[20],"assessing":[21],"quality":[23,100,183],"of":[24,65,106,124,181],"trainer":[25,151,173],"and":[27,54,80,114,172,188],"its":[28],"effectiveness":[29],"influencing":[31],"behavior":[33],"during":[34],"live":[35,141],"surgery":[36],"remains":[37],"challenge.":[39],"Prior":[40],"studies":[41],"assessed":[42],"content":[44],"relying":[45],"on":[46,56,149],"extensive":[47],"manual":[48],"annotation":[49],"expert":[51],"human":[52,125],"raters":[53],"focused":[55],"developing":[57],"broad":[58],"taxonomies":[59],"that":[60,96,155],"overlook":[61],"qualitative":[63],"aspects":[64],"delivery":[67],"such":[68],"as":[69],"clarity":[70],"or":[71],"urgency.":[72],"Limited":[73],"existing":[74],"automated":[75],"methods,":[76],"including":[77,167],"keyword":[78],"analysis":[79],"topic":[81],"modeling,":[82],"also":[83],"fail":[84],"to":[85,119,138],"capture":[86],"these":[87],"nuanced":[88],"aspects.":[89],"We":[90],"introduce":[91],"two-stage":[93],"LLM-based":[94],"framework":[95],"discovers":[97],"interpretable":[98,126],"criteria":[101,128,134,158],"grounded":[102],"context":[105],"surgical":[107,115,142,194],"training.":[108],"Our":[109],"method":[110],"uses":[111],"multi-agent":[112],"prompting":[113],"domain":[116],"knowledge":[117],"injection":[118],"discover":[120],"small":[122],"set":[123],"scoring":[127],"(e.g.,":[129],"Encouraging,":[130],"Urgent,":[131],"Clear).":[132],"These":[133],"are":[135],"then":[136],"used":[137],"automatically":[139],"score":[140],"via":[144],"an":[145],"LLM-as-a-judge":[146],"approach.":[147],"Evaluation":[148],"4.2k":[150],"instances":[153],"demonstrates":[154],"our":[156],"AI-discovered":[157],"outperform":[159],"prior":[160],"content-based":[161],"frameworks":[162],"predicting":[164],"effectiveness,":[166],"observed":[168],"behavioral":[170],"adjustments":[171],"approval.":[174],"This":[175],"work":[176],"advances":[177],"scalable,":[178],"human-aligned":[179],"assessment":[180],"communication":[182],"provides":[189],"foundation":[191],"for":[192],"improving":[193],"teaching":[195],"practices.":[196]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-05-27T00:00:00"}