{"id":"https://openalex.org/W4404261122","doi":"https://doi.org/10.1145/3673791.3698411","title":"Optimizing LLMs with Direct Preferences: A Data Efficiency Perspective","display_name":"Optimizing LLMs with Direct Preferences: A Data Efficiency Perspective","publication_year":2024,"publication_date":"2024-12-08","ids":{"openalex":"https://openalex.org/W4404261122","doi":"https://doi.org/10.1145/3673791.3698411"},"language":"en","primary_location":{"id":"doi:10.1145/3673791.3698411","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698411","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698411","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698411","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114601870","display_name":"Pietro Bernardelle","orcid":null},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Pietro Bernardelle","raw_affiliation_strings":["The University of Queensland, Brisbane, Australia"],"raw_orcid":"https://orcid.org/0009-0003-3657-9229","affiliations":[{"raw_affiliation_string":"The University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052565959","display_name":"Gianluca Demartini","orcid":"https://orcid.org/0000-0002-7311-3693"},"institutions":[{"id":"https://openalex.org/I165143802","display_name":"The University of Queensland","ror":"https://ror.org/00rqy9422","country_code":"AU","type":"education","lineage":["https://openalex.org/I165143802"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Gianluca Demartini","raw_affiliation_strings":["The University of Queensland, Brisbane, Australia"],"raw_orcid":"https://orcid.org/0000-0002-7311-3693","affiliations":[{"raw_affiliation_string":"The University of Queensland, Brisbane, Australia","institution_ids":["https://openalex.org/I165143802"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I165143802"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"236","last_page":"240"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.7344607710838318},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7076533436775208},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6252260208129883},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6013354659080505},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5817217826843262},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5348901152610779},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.45198312401771545},{"id":"https://openalex.org/keywords/judgement","display_name":"Judgement","score":0.43921053409576416},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07968232035636902},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.078244149684906}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.7344607710838318},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7076533436775208},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6252260208129883},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6013354659080505},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5817217826843262},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5348901152610779},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.45198312401771545},{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.43921053409576416},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07968232035636902},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.078244149684906},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3673791.3698411","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698411","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698411","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2410.16586","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.16586","pdf_url":"https://arxiv.org/pdf/2410.16586","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3673791.3698411","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698411","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698411","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.4399999976158142}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4404261122.pdf"},"referenced_works_count":2,"referenced_works":["https://openalex.org/W4226278401","https://openalex.org/W4378771755"],"related_works":["https://openalex.org/W2937325523","https://openalex.org/W4403346496","https://openalex.org/W4205377104","https://openalex.org/W257970033","https://openalex.org/W1994181006","https://openalex.org/W2911102221","https://openalex.org/W2943672508","https://openalex.org/W4285602503","https://openalex.org/W4383737174","https://openalex.org/W4320918405"],"abstract_inverted_index":{"Aligning":[0],"the":[1,38,60,95,111,128,144,158],"output":[2],"of":[3,14,40,43,66,83,97,103,114,130,146,160,163,179],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"with":[8,17,100,183,189],"human":[9,18],"preferences":[10],"(e.g.,":[11],"by":[12],"means":[13],"reinforcement":[15],"learning":[16],"feedback,":[19],"or":[20],"RLHF)":[21],"is":[22,87],"essential":[23],"for":[24,127,134,149],"ensuring":[25],"their":[26,78],"effectiveness":[27,65,119],"in":[28,34,71,120],"real-world":[29],"scenarios.":[30],"Despite":[31],"significant":[32],"advancements":[33],"LLM":[35],"alignment":[36],"techniques,":[37],"impact":[39],"different":[41,177],"type":[42],"preference":[44,84,106,136],"data":[45,62,137,147],"on":[46,80],"model":[47,155,168],"performance":[48,96],"has":[49],"yet":[50],"to":[51,76,89,109],"be":[52],"systematically":[53,93],"explored.":[54],"In":[55],"this":[56],"study,":[57],"we":[58],"investigate":[59],"scalability,":[61],"efficiency,":[63],"and":[64,116,123,153],"Direct":[67],"Preference":[68],"Optimization":[69],"(DPO)":[70],"fine-tuning":[72],"pre-trained":[73],"LLMs,":[74],"aiming":[75],"reduce":[77],"dependency":[79],"extensive":[81],"amounts":[82],"data,":[85],"which":[86],"expensive":[88],"collect.":[90],"We":[91],"(1)":[92],"compare":[94],"models":[98,172,181],"fine-tuned":[99],"varying":[101],"percentages":[102],"a":[104,161],"combined":[105],"judgement":[107],"dataset":[108],"define":[110],"improvement":[112],"curve":[113],"DPO":[115],"assess":[117],"its":[118],"data-constrained":[121],"environments;":[122],"(2)":[124],"provide":[125],"insights":[126],"development":[129],"an":[131],"optimal":[132],"approach":[133],"selective":[135],"usage.":[138],"Our":[139],"study":[140],"reveals":[141],"that":[142],"increasing":[143],"amount":[145],"used":[148],"training":[150],"generally":[151],"enhances":[152],"stabilizes":[154],"performance.":[156],"Moreover,":[157],"use":[159],"combination":[162],"diverse":[164],"datasets":[165],"significantly":[166],"improves":[167],"effectiveness.":[169],"Furthermore,":[170],"when":[171],"are":[173],"trained":[174,182,188],"separately":[175],"using":[176],"types":[178],"prompts,":[180],"conversational":[184],"prompts":[185],"outperformed":[186],"those":[187],"question":[190],"answering":[191],"prompts.":[192]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}