{"id":"https://openalex.org/W4411996505","doi":"https://doi.org/10.1109/access.2025.3585414","title":"Ensemble-Based Uncertainty Quantification for Reliable Large Language Model Classification in Social Data Applications","display_name":"Ensemble-Based Uncertainty Quantification for Reliable Large Language Model Classification in Social Data Applications","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4411996505","doi":"https://doi.org/10.1109/access.2025.3585414"},"language":"en","primary_location":{"id":"doi:10.1109/access.2025.3585414","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3585414","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2025.3585414","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042098061","display_name":"David F. Farr","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David T. Farr","raw_affiliation_strings":["Information School, University of Washington, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0009-0007-8842-7314","affiliations":[{"raw_affiliation_string":"Information School, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001523273","display_name":"Lynnette Hui Xian Ng","orcid":"https://orcid.org/0000-0002-2740-7818"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lynnette Hui Xian Ng","raw_affiliation_strings":["School of Computer Science, Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-2740-7818","affiliations":[{"raw_affiliation_string":"School of Computer Science, Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014652241","display_name":"Iain J. Cruickshank","orcid":"https://orcid.org/0000-0002-4205-5806"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Iain J. Cruickshank","raw_affiliation_strings":["School of Computer Science, Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-4205-5806","affiliations":[{"raw_affiliation_string":"School of Computer Science, Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114337839","display_name":"Nico Manzonelli","orcid":null},"institutions":[{"id":"https://openalex.org/I192545095","display_name":"United States Military Academy","ror":"https://ror.org/01jepya76","country_code":"US","type":"education","lineage":["https://openalex.org/I1304082316","https://openalex.org/I1330347796","https://openalex.org/I192545095","https://openalex.org/I4210088792"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nico Manzonelli","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science, United States Military Academy at West Point, West Point, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science, United States Military Academy at West Point, West Point, NY, USA","institution_ids":["https://openalex.org/I192545095"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034948347","display_name":"Nicholas A. Clark","orcid":"https://orcid.org/0000-0003-0105-9605"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas Clark","raw_affiliation_strings":["Information School, University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information School, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045494255","display_name":"Kate Starbird","orcid":"https://orcid.org/0000-0003-1661-4608"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kate Starbird","raw_affiliation_strings":["Human Centered Design and Engineering, University of Washington, Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0003-1661-4608","affiliations":[{"raw_affiliation_string":"Human Centered Design and Engineering, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032194186","display_name":"Nathaniel D. Bastian","orcid":"https://orcid.org/0000-0001-9957-2778"},"institutions":[{"id":"https://openalex.org/I192545095","display_name":"United States Military Academy","ror":"https://ror.org/01jepya76","country_code":"US","type":"education","lineage":["https://openalex.org/I1304082316","https://openalex.org/I1330347796","https://openalex.org/I192545095","https://openalex.org/I4210088792"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nathaniel D. Bastian","raw_affiliation_strings":["Department of Electrical Engineering and Computer Science, United States Military Academy at West Point, West Point, NY, USA"],"raw_orcid":"https://orcid.org/0000-0001-9957-2778","affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering and Computer Science, United States Military Academy at West Point, West Point, NY, USA","institution_ids":["https://openalex.org/I192545095"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046879461","display_name":"Jevin D. West","orcid":"https://orcid.org/0000-0002-4118-0322"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jevin West","raw_affiliation_strings":["Information School, University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information School, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":1.7588,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.87305329,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"13","issue":null,"first_page":"116419","last_page":"116429"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9714000225067139,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9610000252723694,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6863312721252441},{"id":"https://openalex.org/keywords/ensemble-learning","display_name":"Ensemble learning","score":0.4459807276725769},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41455626487731934},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3723890781402588},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3515547513961792}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6863312721252441},{"id":"https://openalex.org/C45942800","wikidata":"https://www.wikidata.org/wiki/Q245652","display_name":"Ensemble learning","level":2,"score":0.4459807276725769},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41455626487731934},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3723890781402588},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3515547513961792}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2025.3585414","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3585414","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:13c7bfc435c1456b814527070ac0ab7d","is_oa":true,"landing_page_url":"https://doaj.org/article/13c7bfc435c1456b814527070ac0ab7d","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 13, Pp 116419-116429 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2025.3585414","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3585414","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":47,"referenced_works":["https://openalex.org/W560287614","https://openalex.org/W2001442493","https://openalex.org/W2130237711","https://openalex.org/W2161920755","https://openalex.org/W2460159515","https://openalex.org/W2566963797","https://openalex.org/W2741212913","https://openalex.org/W2884295076","https://openalex.org/W2986889260","https://openalex.org/W3047027807","https://openalex.org/W3099474967","https://openalex.org/W3214119601","https://openalex.org/W4285301760","https://openalex.org/W4294597242","https://openalex.org/W4295872590","https://openalex.org/W4380301730","https://openalex.org/W4384200891","https://openalex.org/W4389518686","https://openalex.org/W4389636360","https://openalex.org/W4392240262","https://openalex.org/W4394780772","https://openalex.org/W4395484085","https://openalex.org/W4396218239","https://openalex.org/W4396809921","https://openalex.org/W4396831993","https://openalex.org/W4400004771","https://openalex.org/W4400055904","https://openalex.org/W4401043565","https://openalex.org/W4401630357","https://openalex.org/W4402827393","https://openalex.org/W4404782678","https://openalex.org/W4404782982","https://openalex.org/W4408812939","https://openalex.org/W4410376226","https://openalex.org/W6730042731","https://openalex.org/W6753084064","https://openalex.org/W6778883912","https://openalex.org/W6801617135","https://openalex.org/W6839548382","https://openalex.org/W6850423384","https://openalex.org/W6851151028","https://openalex.org/W6853906918","https://openalex.org/W6857598455","https://openalex.org/W6862111818","https://openalex.org/W6874219447","https://openalex.org/W6874851699","https://openalex.org/W6875092577"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Assessing":[0],"classification":[1,40,60],"confidence":[2],"is":[3],"essential":[4],"for":[5],"effectively":[6],"leveraging":[7],"Large":[8],"Language":[9],"Models":[10],"(LLMs)":[11],"in":[12,105],"automated":[13],"data":[14,92,136],"labeling,":[15],"particularly":[16],"within":[17],"the":[18,122,130],"sensitive":[19],"contexts":[20],"of":[21,108,117,124,134],"Computational":[22],"Social":[23],"Science":[24],"(CSS)":[25],"tasks.":[26],"In":[27],"this":[28],"study,":[29],"we":[30,62],"evaluate":[31],"five":[32],"uncertainty":[33,87],"quantification":[34],"(UQ)":[35],"strategies":[36,52],"across":[37],"three":[38,54],"CSS":[39],"problems:":[41],"stance":[42],"detection,":[43],"ideology":[44],"identification,":[45],"and":[46,72,132],"frame":[47],"detection.":[48],"We":[49],"benchmark":[50],"these":[51],"using":[53],"different":[55],"LLMs.":[56],"To":[57],"enhance":[58],"human-in-the-loop":[59,135],"performance,":[61],"introduce":[63],"an":[64,113],"ensemble-based":[65],"UQ":[66,103],"aggregation":[67],"method,":[68],"<italic":[69,78,98],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[70,79,99],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">C_ensemble</i>,":[71],"propose":[73],"a":[74],"novel":[75],"evaluation":[76],"metric,":[77],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Misclassified":[80],"Recall</i>,":[81],"designed":[82],"to":[83,127],"better":[84],"assess":[85],"model":[86],"on":[88],"mislabeled":[89],"or":[90],"ambiguous":[91],"points.":[93],"Our":[94],"results":[95],"show":[96],"that":[97],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">C_ensemble</i>":[100],"outperforms":[101],"existing":[102],"techniques":[104],"six":[106],"out":[107],"nine":[109],"model\u2013dataset":[110],"combinations,":[111],"achieving":[112],"average":[114],"AUC":[115],"improvement":[116],"8.7.":[118],"These":[119],"findings":[120],"highlight":[121],"potential":[123],"UQ-driven":[125],"methods":[126],"significantly":[128],"improve":[129],"reliability":[131],"efficiency":[133],"annotation":[137],"pipelines.":[138]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-07-04T00:00:00"}