{"id":"https://openalex.org/W4389518896","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.608","title":"Democratizing LLMs: An Exploration of Cost-Performance Trade-offs in Self-Refined Open-Source Models","display_name":"Democratizing LLMs: An Exploration of Cost-Performance Trade-offs in Self-Refined Open-Source Models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389518896","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.608"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.608","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.608","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.608","pdf_url":"https://aclanthology.org/2023.findings-emnlp.608.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.findings-emnlp.608.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040980466","display_name":"Sumuk Shashidhar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sumuk Shashidhar","raw_affiliation_strings":["Department of Computer Science, University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032691039","display_name":"Abhinav Chinta","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abhinav Chinta","raw_affiliation_strings":["Department of Computer Science, University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076767039","display_name":"Vaibhav Sahai","orcid":"https://orcid.org/0000-0003-1892-1548","observed_orcids":["https://orcid.org/0000-0003-1892-1548"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vaibhav Sahai","raw_affiliation_strings":["Department of Computer Science, University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005802610","display_name":"Zhenhailong Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhenhailong Wang","raw_affiliation_strings":["Department of Computer Science, University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103178893","display_name":"Heng Ji","orcid":"https://orcid.org/0000-0002-7954-7994","observed_orcids":["https://orcid.org/0000-0002-7954-7994"]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Heng Ji","raw_affiliation_strings":["Department of Computer Science, University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":2.2668,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":{"value":0.92026416,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"9070","last_page":"9084"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.926800012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.926800012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05770000070333481,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.002199999988079071,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-refinement","display_name":"self-refinement","score":0.7900000214576721},{"id":"https://openalex.org/keywords/self-criticism","display_name":"self-criticism","score":0.8939999938011169},{"id":"https://openalex.org/keywords/open-source-language-models","display_name":"open-source language models","score":0.8980000019073486},{"id":"https://openalex.org/keywords/cost-performance-trade-off","display_name":"cost-performance trade-off","score":0.8949999809265137},{"id":"https://openalex.org/keywords/model-selection","display_name":"model selection","score":0.9200000166893005},{"id":"https://openalex.org/keywords/vicuna","display_name":"vicu\u00f1a","score":0.6930000185966492},{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.5450000166893005},{"id":"https://openalex.org/keywords/democratization-of-ai","display_name":"democratization of AI","score":0.48899999260902405},{"id":"https://openalex.org/keywords/data-privacy","display_name":"data privacy","score":0.7239999771118164},{"id":"https://openalex.org/keywords/resource-limited-environments","display_name":"resource-constrained environments","score":0.6309999823570251}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7229056358337402},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6223909854888916},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.5466369986534119},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5410702228546143},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.48758572340011597},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4329748749732971},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3265276849269867},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.1185251772403717},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.11258149147033691},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10929197072982788},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.findings-emnlp.608","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.608","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.608","pdf_url":"https://aclanthology.org/2023.findings-emnlp.608.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2310.07611","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.07611","pdf_url":"https://arxiv.org/pdf/2310.07611","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.07611","doi":"https://doi.org/10.48550/arxiv.2310.07611","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.07611","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.608","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.608","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.findings-emnlp.608","pdf_url":"https://aclanthology.org/2023.findings-emnlp.608.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389518896.pdf","grobid_xml":"https://content.openalex.org/works/W4389518896.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W2560647685","https://openalex.org/W2794325560","https://openalex.org/W2946609015","https://openalex.org/W3121904249","https://openalex.org/W3168867926","https://openalex.org/W3201174429","https://openalex.org/W4224308101","https://openalex.org/W4307079201","https://openalex.org/W4322718191","https://openalex.org/W4377372007","https://openalex.org/W4385571232","https://openalex.org/W4385572239","https://openalex.org/W4389344855","https://openalex.org/W4389523706","https://openalex.org/W4389524426","https://openalex.org/W4401043863","https://openalex.org/W7133222210","https://openalex.org/W7133224126","https://openalex.org/W7133225605","https://openalex.org/W7133233472","https://openalex.org/W7133236450","https://openalex.org/W7133239970"],"related_works":["https://openalex.org/W4402671190","https://openalex.org/W7133236450","https://openalex.org/W7108211030","https://openalex.org/W7154891330","https://openalex.org/W7128566517","https://openalex.org/W7055922026","https://openalex.org/W7172420567","https://openalex.org/W7147468801","https://openalex.org/W4394780940","https://openalex.org/W4388183168"],"abstract_inverted_index":{"The":[0],"dominance":[1],"of":[2,39,45,80],"proprietary":[3],"LLMs":[4,145],"has":[5,134],"led":[6],"to":[7,111,143,176],"restricted":[8],"access":[9,175],"and":[10,21,42,54,71,109,129,139,153,173],"raised":[11],"information":[12],"privacy":[13],"concerns.High-performing":[14],"open-source":[15],"alternatives":[16],"are":[17],"crucial":[18],"for":[19,64,137],"information-sensitive":[20],"high-volume":[22],"applications":[23],"but":[24],"often":[25],"lag":[26],"behind":[27],"in":[28,115,167],"performance.To":[29],"address":[30],"this":[31],"gap,":[32],"we":[33],"propose":[34],"(1)":[35],"A":[36,48],"untargeted":[37],"variant":[38],"iterative":[40],"selfcritique":[41],"self-refinement":[43,156],"devoid":[44],"external":[46],"influence.(2)":[47],"novel":[49,161],"ranking":[50,162],"metric":[51,163],"-Performance,":[52],"Refinement,":[53],"Inference":[55],"Cost":[56],"Score":[57],"(PeRFICS)":[58],"-to":[59],"find":[60],"the":[61,121],"optimal":[62],"model":[63,168],"a":[65,105,112,126],"given":[66],"task":[67],"considering":[68],"refined":[69],"performance":[70,152],"cost.Our":[72],"experiments":[73],"show":[74,104],"that":[75],"SoTA":[76],"open":[77,117],"source":[78],"models":[79,95],"varying":[81],"sizes":[82],"from":[83,90],"7B":[84],"-65B,":[85],"on":[86,120,151],"average,":[87],"improve":[88],"8.2%":[89],"their":[91],"baseline":[92],"performance.Strikingly,":[93],"even":[94],"with":[96,159],"extremely":[97],"small":[98],"memory":[99],"footprints,":[100],"such":[101],"as":[102,180],"Vicuna-7B,":[103],"11.74%":[106],"improvement":[107,114],"overall":[108],"up":[110],"25.39%":[113],"high-creativity,":[116],"ended":[118],"tasks":[119],"Vicuna":[122],"benchmark.Vicuna-13B":[123],"takes":[124],"it":[125],"step":[127],"further":[128],"outperforms":[130],"ChatGPT":[131],"postrefinement.This":[132],"work":[133],"profound":[135],"implications":[136],"resource-constrained":[138],"informationsensitive":[140],"environments":[141],"seeking":[142],"leverage":[144],"without":[146],"incurring":[147],"prohibitive":[148],"costs,":[149],"compromising":[150],"privacy.The":[154],"domain-agnostic":[155],"process":[157],"coupled":[158],"our":[160],"facilitates":[164],"informed":[165],"decision-making":[166],"selection,":[169],"thereby":[170],"reducing":[171],"costs":[172],"democratizing":[174],"high-performing":[177],"language":[178],"models,":[179],"evidenced":[181],"by":[182],"case":[183],"studies.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":5}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-12-10T00:00:00"}