{"id":"https://openalex.org/W4406459039","doi":"https://doi.org/10.1109/bigdata62323.2024.10826109","title":"LLM Chain Ensembles for Scalable and Accurate Data Annotation","display_name":"LLM Chain Ensembles for Scalable and Accurate Data Annotation","publication_year":2024,"publication_date":"2024-12-15","ids":{"openalex":"https://openalex.org/W4406459039","doi":"https://doi.org/10.1109/bigdata62323.2024.10826109"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata62323.2024.10826109","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10826109","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042098061","display_name":"David F. Farr","orcid":null},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"David Farr","raw_affiliation_strings":["University of Washington,Seattle,WA"],"affiliations":[{"raw_affiliation_string":"University of Washington,Seattle,WA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114337784","display_name":"Nico Manzonelli","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nico Manzonelli","raw_affiliation_strings":["Army Cyber Technology and Innovation Center,Augusta,GA"],"affiliations":[{"raw_affiliation_string":"Army Cyber Technology and Innovation Center,Augusta,GA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050725358","display_name":"Iain Cruickshank","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Iain Cruickshank","raw_affiliation_strings":["Carnegie Mellon University,Pittsburgh,PA"],"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,Pittsburgh,PA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045494255","display_name":"Kate Starbird","orcid":"https://orcid.org/0000-0003-1661-4608"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kate Starbird","raw_affiliation_strings":["University of Washington,Seattle,WA"],"affiliations":[{"raw_affiliation_string":"University of Washington,Seattle,WA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046879461","display_name":"Jevin D. West","orcid":"https://orcid.org/0000-0002-4118-0322"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jevin West","raw_affiliation_strings":["University of Washington,Seattle,WA"],"affiliations":[{"raw_affiliation_string":"University of Washington,Seattle,WA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5042098061"],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":1.0878,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.82761022,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"2110","last_page":"2118"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.752392053604126},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6291916370391846},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.6254834532737732},{"id":"https://openalex.org/keywords/chain","display_name":"Chain (unit)","score":0.4942944049835205},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37875795364379883},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.11267223954200745}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.752392053604126},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6291916370391846},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6254834532737732},{"id":"https://openalex.org/C199185054","wikidata":"https://www.wikidata.org/wiki/Q552299","display_name":"Chain (unit)","level":2,"score":0.4942944049835205},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37875795364379883},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.11267223954200745},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata62323.2024.10826109","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10826109","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":52,"referenced_works":["https://openalex.org/W1532008752","https://openalex.org/W1593114658","https://openalex.org/W2130237711","https://openalex.org/W2150884987","https://openalex.org/W2161920755","https://openalex.org/W2910453440","https://openalex.org/W3004975108","https://openalex.org/W3011573535","https://openalex.org/W3205301323","https://openalex.org/W4205737716","https://openalex.org/W4205807230","https://openalex.org/W4221143046","https://openalex.org/W4285537247","https://openalex.org/W4366733439","https://openalex.org/W4376122773","https://openalex.org/W4385571189","https://openalex.org/W4387075633","https://openalex.org/W4387163503","https://openalex.org/W4389518684","https://openalex.org/W4389518837","https://openalex.org/W4392051461","https://openalex.org/W4392426167","https://openalex.org/W4394579953","https://openalex.org/W4395443209","https://openalex.org/W4399205331","https://openalex.org/W4400023563","https://openalex.org/W4401042461","https://openalex.org/W4401042932","https://openalex.org/W4401043635","https://openalex.org/W4403753188","https://openalex.org/W4403939562","https://openalex.org/W4404783774","https://openalex.org/W4406024033","https://openalex.org/W6615576869","https://openalex.org/W6635221813","https://openalex.org/W6718565325","https://openalex.org/W6809646742","https://openalex.org/W6843187757","https://openalex.org/W6852625459","https://openalex.org/W6852670370","https://openalex.org/W6852805694","https://openalex.org/W6856404938","https://openalex.org/W6856459790","https://openalex.org/W6857855513","https://openalex.org/W6860710830","https://openalex.org/W6861882308","https://openalex.org/W6865013197","https://openalex.org/W6869117115","https://openalex.org/W6873165011","https://openalex.org/W6873417195","https://openalex.org/W6875092577","https://openalex.org/W6878433926"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328"],"abstract_inverted_index":{"The":[0],"ability":[1],"of":[2,37,73,115],"large":[3],"language":[4],"models":[5,63],"(LLMs)":[6],"to":[7,31,61,83,98],"perform":[8],"zero-shot":[9],"classification":[10,66],"makes":[11],"them":[12],"viable":[13],"solutions":[14],"for":[15,137],"data":[16,25,59,85,139],"annotation":[17,140],"in":[18,55,120],"rapidly":[19],"evolving":[20],"domains":[21],"where":[22,87],"quality":[23],"labeled":[24],"is":[26],"often":[27,111],"scarce":[28],"and":[29,123,134],"costly":[30],"obtain.":[32],"However,":[33],"the":[34,71,90,107,113,116,121],"large-scale":[35,138],"deployment":[36],"LLMs":[38,54,75],"can":[39],"be":[40],"prohibitively":[41],"expensive.":[42],"This":[43,68],"paper":[44],"introduces":[45],"an":[46],"LLM":[47,129],"chain":[48,108,122,130],"ensemble":[49,109],"methodology":[50],"that":[51,106],"aligns":[52],"multiple":[53],"a":[56,77,132],"sequence,":[57],"routing":[58],"subsets":[60],"subsequent":[62],"based":[64],"on":[65],"uncertainty.":[67],"approach":[69],"leverages":[70],"strengths":[72],"individual":[74,118],"within":[76],"broader":[78],"system,":[79],"allowing":[80],"each":[81],"model":[82,119],"handle":[84],"points":[86],"it":[88],"exhibits":[89],"highest":[91],"confidence,":[92],"while":[93],"forwarding":[94],"more":[95,100],"complex":[96],"cases":[97],"potentially":[99],"robust":[101],"models.":[102],"Our":[103],"results":[104],"show":[105],"method":[110],"exceeds":[112],"performance":[114],"best":[117],"achieves":[124],"substantial":[125],"cost":[126],"savings,":[127],"making":[128],"ensembles":[131],"practical":[133],"efficient":[135],"solution":[136],"challenges.":[141]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}