{"id":"https://openalex.org/W7130600932","doi":"https://doi.org/10.1109/fllm67465.2025.11391029","title":"Tuning LLM-Based Advisors for the Common Good: The Case for Direct Preference Optimization","display_name":"Tuning LLM-Based Advisors for the Common Good: The Case for Direct Preference Optimization","publication_year":2025,"publication_date":"2025-11-25","ids":{"openalex":"https://openalex.org/W7130600932","doi":"https://doi.org/10.1109/fllm67465.2025.11391029"},"language":"en","primary_location":{"id":"doi:10.1109/fllm67465.2025.11391029","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fllm67465.2025.11391029","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 3rd International Conference on Foundation and Large Language Models (FLLM)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hdl.handle.net/2434/1233599","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058810729","display_name":"Lara Mauri","orcid":"https://orcid.org/0000-0002-4024-1015"},"institutions":[{"id":"https://openalex.org/I189158943","display_name":"University of Milan","ror":"https://ror.org/00wjc7c48","country_code":"IT","type":"education","lineage":["https://openalex.org/I189158943"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Lara Mauri","raw_affiliation_strings":["Università Degli Studi di Milano,Department of Computer Science,Milan,Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Università Degli Studi di Milano,Department of Computer Science,Milan,Italy","institution_ids":["https://openalex.org/I189158943"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022116106","display_name":"Gohar Sargsyan","orcid":null},"institutions":[{"id":"https://openalex.org/I55215948","display_name":"Tata Consultancy Services (India)","ror":"https://ror.org/01b9n8m42","country_code":"IN","type":"company","lineage":["https://openalex.org/I4210086519","https://openalex.org/I55215948"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Gohar Sargsyan","raw_affiliation_strings":["Tata Consultancy Services, Europe"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tata Consultancy Services, Europe","institution_ids":["https://openalex.org/I55215948"]}]},{"author_position":"last","author":{"id":null,"display_name":"Ernesto Damiani","orcid":null},"institutions":[{"id":"https://openalex.org/I189158943","display_name":"University of Milan","ror":"https://ror.org/00wjc7c48","country_code":"IT","type":"education","lineage":["https://openalex.org/I189158943"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Ernesto Damiani","raw_affiliation_strings":["Università Degli Studi di Milano,Department of Computer Science,Milan,Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Università Degli Studi di Milano,Department of Computer Science,Milan,Italy","institution_ids":["https://openalex.org/I189158943"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"910","last_page":"915"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.2874000072479248,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.2874000072479248,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.1648000031709671,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.053199999034404755,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.6955999732017517},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6442999839782715},{"id":"https://openalex.org/keywords/advice","display_name":"Advice (programming)","score":0.5644999742507935},{"id":"https://openalex.org/keywords/sustainability","display_name":"Sustainability","score":0.5443000197410583},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.44940000772476196},{"id":"https://openalex.org/keywords/carbon-footprint","display_name":"Carbon footprint","score":0.4392000138759613},{"id":"https://openalex.org/keywords/compliance","display_name":"Compliance (psychology)","score":0.42089998722076416}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6955999732017517},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6442999839782715},{"id":"https://openalex.org/C2779955035","wikidata":"https://www.wikidata.org/wiki/Q4686785","display_name":"Advice (programming)","level":2,"score":0.5644999742507935},{"id":"https://openalex.org/C66204764","wikidata":"https://www.wikidata.org/wiki/Q219416","display_name":"Sustainability","level":2,"score":0.5443000197410583},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5145999789237976},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.46129998564720154},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.44940000772476196},{"id":"https://openalex.org/C2780936489","wikidata":"https://www.wikidata.org/wiki/Q310667","display_name":"Carbon footprint","level":3,"score":0.4392000138759613},{"id":"https://openalex.org/C2781460075","wikidata":"https://www.wikidata.org/wiki/Q1399332","display_name":"Compliance (psychology)","level":2,"score":0.42089998722076416},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4052000045776367},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.35760000348091125},{"id":"https://openalex.org/C132943942","wikidata":"https://www.wikidata.org/wiki/Q2562511","display_name":"Footprint","level":2,"score":0.3398999869823456},{"id":"https://openalex.org/C52820260","wikidata":"https://www.wikidata.org/wiki/Q7551312","display_name":"Social preferences","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.3084000051021576},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.30630001425743103},{"id":"https://openalex.org/C83516724","wikidata":"https://www.wikidata.org/wiki/Q234173","display_name":"Ecological footprint","level":3,"score":0.2831999957561493},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.28279998898506165},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2784000039100647},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2728999853134155},{"id":"https://openalex.org/C134560507","wikidata":"https://www.wikidata.org/wiki/Q753291","display_name":"Environmental economics","level":1,"score":0.2711000144481659},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.25440001487731934}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/fllm67465.2025.11391029","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fllm67465.2025.11391029","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 3rd International Conference on Foundation and Large Language Models (FLLM)","raw_type":"proceedings-article"},{"id":"pmh:oai:air.unimi.it:2434/1233599","is_oa":true,"landing_page_url":"https://hdl.handle.net/2434/1233599","pdf_url":null,"source":{"id":"https://openalex.org/S4306400516","display_name":"Archivio Istituzionale della Ricerca (Universita Degli Studi Di Milano)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I189158943","host_organization_name":"University of Milan","host_organization_lineage":["https://openalex.org/I189158943"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/bookPart"}],"best_oa_location":{"id":"pmh:oai:air.unimi.it:2434/1233599","is_oa":true,"landing_page_url":"https://hdl.handle.net/2434/1233599","pdf_url":null,"source":{"id":"https://openalex.org/S4306400516","display_name":"Archivio Istituzionale della Ricerca (Universita Degli Studi Di Milano)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I189158943","host_organization_name":"University of Milan","host_organization_lineage":["https://openalex.org/I189158943"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/bookPart"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320313475","display_name":"Universit\u00e0 degli Studi di Milano","ror":"https://ror.org/00wjc7c48"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W2767983543","https://openalex.org/W2923014074","https://openalex.org/W3046375318","https://openalex.org/W3102483398","https://openalex.org/W3204112174","https://openalex.org/W4221100501","https://openalex.org/W4378189609","https://openalex.org/W4385572697","https://openalex.org/W4401042735","https://openalex.org/W4413175758","https://openalex.org/W4413680113","https://openalex.org/W4415207023"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4],"often":[5],"deployed":[6],"as":[7],"advisors":[8],"to":[9,15,41,52,86,93,104,143],"consumers,":[10],"e.g.,":[11,17],"recommending":[12,39,48],"purchases,":[13],"and":[14,65,100,117,119,150,163],"managers,":[16],"suggesting":[18],"new":[19,50],"hires.":[20],"In":[21],"fact,":[22],"LLMs":[23],"provide":[24],"advice":[25,58],"based":[26,125],"on":[27,126],"cost":[28],"or":[29,45],"convenience,":[30],"overlooking":[31],"broader":[32],"societal":[33,60],"impacts":[34],"(e.g.,":[35],"carbon":[36],"footprint":[37],"when":[38,47],"products":[40],"a":[42,49,53,141],"potential":[43],"customer,":[44],"fairness":[46],"hire":[51],"company).":[54],"To":[55],"align":[56],"LLMs'":[57],"with":[59],"goals":[61],"like":[62],"environmental":[63],"sustainability":[64],"gender":[66],"parity,":[67],"tuning":[68,82,112,124,159],"strategies":[69],"must":[70],"integrate":[71],"the":[72,110],"notion":[73],"of":[74,132],"common":[75],"good.":[76],"We":[77,139],"discuss":[78],"why":[79],"Direct":[80,105],"Alignment":[81],"could":[83],"be":[84],"preferable":[85],"classic":[87],"Reinforcement":[88],"Learning":[89],"from":[90,115],"Human":[91],"Feedback":[92],"achieve":[94],"this":[95],"integration.":[96],"Then,":[97],"we":[98],"describe":[99],"compare":[101],"two":[102],"approaches":[103],"Preference":[106],"Optimization:":[107],"(1)":[108],"exposing":[109],"model":[111,123],"examples":[113],"taken":[114],"recommendations":[116],"regulations,":[118],"(2)":[120],"mythopoiesis,":[121],"i.e.,":[122],"synthetic":[127],"\"legends\",":[128],"fictional":[129],"success":[130],"stories":[131],"regulatory":[133],"compliance":[134],"(also":[135],"generated":[136],"by":[137],"LLMs).":[138],"present":[140],"pipeline":[142],"evaluate":[144],"legends'":[145],"effectiveness":[146],"in":[147],"reducing":[148],"bias":[149],"fostering":[151],"compliance.":[152],"Our":[153],"preliminary":[154],"results":[155],"suggest":[156],"that":[157],"legend-based":[158],"may":[160],"enhance":[161],"engagement":[162],"generalization,":[164],"while":[165],"direct":[166],"exposure":[167],"ensures":[168],"factual":[169],"accuracy":[170],"but":[171],"risks":[172],"rigidity.":[173]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-02-20T00:00:00"}