{"id":"https://openalex.org/W7166715074","doi":"https://doi.org/10.48550/arxiv.2606.30412","title":"Can LLMs Rank? A Tale of Triads and Triage","display_name":"Can LLMs Rank? A Tale of Triads and Triage","publication_year":2026,"publication_date":"2026-06-29","ids":{"openalex":"https://openalex.org/W7166715074","doi":"https://doi.org/10.48550/arxiv.2606.30412"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.30412","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30412","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.30412","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043254015","display_name":"Gaurab Pokharel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pokharel, Gaurab","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139648009","display_name":"Shafkat Farabi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Farabi, Shafkat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032825437","display_name":"Patrick J. Fowler","orcid":"https://orcid.org/0000-0002-3941-3916"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fowler, Patrick J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5005981376","display_name":"S R Das","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Das, Sanmay","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11464","display_name":"Homelessness and Social Issues","score":0.16210000216960907,"subfield":{"id":"https://openalex.org/subfields/3600","display_name":"General Health Professions"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11464","display_name":"Homelessness and Social Issues","score":0.16210000216960907,"subfield":{"id":"https://openalex.org/subfields/3600","display_name":"General Health Professions"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11502","display_name":"Facility Location and Emergency Management","score":0.11699999868869781,"subfield":{"id":"https://openalex.org/subfields/1407","display_name":"Organizational Behavior and Human Resource Management"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11500","display_name":"Evacuation and Crowd Dynamics","score":0.06889999657869339,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7645000219345093},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.7231000065803528},{"id":"https://openalex.org/keywords/triage","display_name":"Triage","score":0.6244999766349792},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5871000289916992},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.58160001039505},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5115000009536743},{"id":"https://openalex.org/keywords/prioritization","display_name":"Prioritization","score":0.38359999656677246},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.37059998512268066}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7645000219345093},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7231000065803528},{"id":"https://openalex.org/C2777120189","wikidata":"https://www.wikidata.org/wiki/Q780067","display_name":"Triage","level":2,"score":0.6244999766349792},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5871000289916992},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.58160001039505},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5115000009536743},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.4230000078678131},{"id":"https://openalex.org/C2777615720","wikidata":"https://www.wikidata.org/wiki/Q11888847","display_name":"Prioritization","level":2,"score":0.38359999656677246},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.37059998512268066},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3621000051498413},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.34880000352859497},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.328900009393692},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.310699999332428},{"id":"https://openalex.org/C2777381055","wikidata":"https://www.wikidata.org/wiki/Q308922","display_name":"Damages","level":2,"score":0.3091000020503998},{"id":"https://openalex.org/C136975688","wikidata":"https://www.wikidata.org/wiki/Q1320634","display_name":"Tournament","level":2,"score":0.30379998683929443},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.2976999878883362},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.29269999265670776},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.29019999504089355},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.2786000072956085},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C2781472820","wikidata":"https://www.wikidata.org/wiki/Q2154759","display_name":"Risk measure","level":3,"score":0.26649999618530273},{"id":"https://openalex.org/C2780724011","wikidata":"https://www.wikidata.org/wiki/Q1295316","display_name":"Emergency department","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.2606000006198883}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.30412","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30412","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.30412","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30412","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"From":[0],"housing":[1],"allocation":[2,181],"for":[3,26,135,158,204,219],"households":[4],"experiencing":[5],"homelessness":[6,179],"to":[7,77,87,109,161,216],"triage":[8],"in":[9,117,145],"emergency":[10,183],"departments,":[11],"LLMs":[12,64,189],"are":[13,84,153],"increasingly":[14],"being":[15],"considered":[16],"as":[17,66],"judges":[18],"of":[19,44,96,104,125,131,164,171,199],"consequential":[20],"decisions":[21],"that":[22,150],"require":[23],"ranking":[24,220],"people":[25],"scarce":[27],"resources.":[28],"Ranking":[29],"large":[30],"groups":[31],"simultaneously":[32],"is":[33],"cognitively":[34],"demanding":[35],"and":[36,51,148,156,182,211],"error-prone.":[37],"A":[38,99],"natural":[39],"solution,":[40],"drawing":[41],"on":[42],"decades":[43],"social":[45],"choice":[46],"theory,":[47],"elicits":[48],"pairwise":[49,68],"comparisons":[50],"aggregates":[52],"them":[53],"into":[54],"a":[55,59,72,78,121,217],"total":[56],"order.":[57],"However,":[58],"fundamental":[60],"question":[61],"remains":[62],"when":[63],"serve":[65],"the":[67,81,89,102,168],"judge:":[69],"how":[70,205],"can":[71,139],"practitioner":[73],"tell,":[74],"before":[75,214],"committing":[76,215],"ranking,":[79],"whether":[80],"LLM's":[82],"judgments":[83],"sufficiently":[85],"consistent":[86],"trust":[88],"result?":[90],"We":[91,143,166,201],"discuss":[92],"two":[93,175,197],"different":[94,187,192],"ways":[95],"identifying":[97],"consistency.":[98,127,200],"classical":[100],"diagnostic,":[101],"coefficient":[103],"consistency":[105,213],"$\u03b6$,":[106],"originally":[107],"developed":[108],"measure":[110,124,140],"judge":[111],"reliability":[112,163],"by":[113],"counting":[114],"circular":[115],"triads":[116],"tournament":[118],"graphs,":[119],"provides":[120],"cheap,":[122],"model-free":[123],"intra-run":[126],"Various":[128],"standard":[129],"measures":[130,152],"distance":[132],"between":[133],"rankings,":[134],"example":[136],"Kendall's":[137],"$\u03c4$,":[138],"inter-run":[141],"variability.":[142],"show,":[144],"both":[146,160],"theory":[147],"practice,":[149],"these":[151,196],"independently":[154],"valuable,":[155],"advocate":[157],"using":[159],"assess":[162],"rankings.":[165],"demonstrate":[167],"practical":[169],"importance":[170],"our":[172],"results":[173],"across":[174,195],"high-stakes":[176],"prioritization":[177],"tasks:":[178],"service":[180],"department":[184],"triage.":[185],"Three":[186],"leading":[188],"have":[190],"considerably":[191],"performance":[193],"profiles":[194],"axes":[198],"provide":[202],"guidelines":[203],"practitioners":[206],"could":[207],"think":[208],"about":[209],"measuring":[210],"assessing":[212],"model":[218],"or":[221],"prioritization.":[222]},"counts_by_year":[],"updated_date":"2026-07-01T06:29:00.853634","created_date":"2026-07-01T00:00:00"}