{"id":"https://openalex.org/W4393970715","doi":"https://doi.org/10.1145/3640543.3645144","title":"ConstitutionMaker: Interactively Critiquing Large Language Models by Converting Feedback into Principles","display_name":"ConstitutionMaker: Interactively Critiquing Large Language Models by Converting Feedback into Principles","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4393970715","doi":"https://doi.org/10.1145/3640543.3645144"},"language":"en","primary_location":{"id":"doi:10.1145/3640543.3645144","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645144","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645144","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645144","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043565372","display_name":"Savvas Petridis","orcid":"https://orcid.org/0000-0002-4944-8477"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Savvas Petridis","raw_affiliation_strings":["Google, United States"],"raw_orcid":"https://orcid.org/0000-0002-4944-8477","affiliations":[{"raw_affiliation_string":"Google, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078819047","display_name":"Ben Wedin","orcid":"https://orcid.org/0009-0003-5389-1230"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Benjamin D Wedin","raw_affiliation_strings":["Google Research, United States"],"raw_orcid":"https://orcid.org/0009-0003-5389-1230","affiliations":[{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081809692","display_name":"James Wexler","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Wexler","raw_affiliation_strings":["Google Research, United States"],"raw_orcid":"https://orcid.org/0009-0006-8105-6998","affiliations":[{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007386215","display_name":"Mahima Pushkarna","orcid":"https://orcid.org/0000-0002-5903-5510"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mahima Pushkarna","raw_affiliation_strings":["Google Research, United States"],"raw_orcid":"https://orcid.org/0000-0002-5903-5510","affiliations":[{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066462000","display_name":"Aaron Donsbach","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aaron Donsbach","raw_affiliation_strings":["Google, United States"],"raw_orcid":"https://orcid.org/0009-0008-5943-5363","affiliations":[{"raw_affiliation_string":"Google, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025175160","display_name":"Nitesh Goyal","orcid":"https://orcid.org/0000-0002-4666-1926"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nitesh Goyal","raw_affiliation_strings":["Google Research, United States"],"raw_orcid":"https://orcid.org/0000-0002-4666-1926","affiliations":[{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065663195","display_name":"Carrie J. Cai","orcid":"https://orcid.org/0000-0001-9421-7128"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carrie J Cai","raw_affiliation_strings":["Google, United States"],"raw_orcid":"https://orcid.org/0000-0001-9421-7128","affiliations":[{"raw_affiliation_string":"Google, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015483503","display_name":"Michael Terry","orcid":"https://orcid.org/0000-0003-1941-939X"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Terry","raw_affiliation_strings":["Google, United States"],"raw_orcid":"https://orcid.org/0000-0003-1941-939X","affiliations":[{"raw_affiliation_string":"Google, United States","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5043565372"],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":7.9472,"has_fulltext":true,"cited_by_count":24,"citation_normalized_percentile":{"value":0.9777887,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"853","last_page":"868"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatbot","display_name":"Chatbot","score":0.9765220284461975},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7806710004806519},{"id":"https://openalex.org/keywords/formative-assessment","display_name":"Formative assessment","score":0.6836047172546387},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6057106852531433},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5883605480194092},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5767673254013062},{"id":"https://openalex.org/keywords/corrective-feedback","display_name":"Corrective feedback","score":0.42544203996658325},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2304384410381317},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.138935387134552},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.07558763027191162}],"concepts":[{"id":"https://openalex.org/C2779041454","wikidata":"https://www.wikidata.org/wiki/Q870780","display_name":"Chatbot","level":2,"score":0.9765220284461975},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7806710004806519},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.6836047172546387},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6057106852531433},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5883605480194092},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5767673254013062},{"id":"https://openalex.org/C2779305910","wikidata":"https://www.wikidata.org/wiki/Q5172809","display_name":"Corrective feedback","level":2,"score":0.42544203996658325},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2304384410381317},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.138935387134552},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.07558763027191162},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3640543.3645144","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645144","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645144","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3640543.3645144","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645144","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645144","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393970715.pdf","grobid_xml":"https://content.openalex.org/works/W4393970715.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W2013164621","https://openalex.org/W2058990114","https://openalex.org/W2157289187","https://openalex.org/W2591921914","https://openalex.org/W2611049140","https://openalex.org/W2761896474","https://openalex.org/W2962796276","https://openalex.org/W3033889389","https://openalex.org/W3094252362","https://openalex.org/W3160638507","https://openalex.org/W3203321135","https://openalex.org/W3206079177","https://openalex.org/W3206368713","https://openalex.org/W3206370442","https://openalex.org/W4220920983","https://openalex.org/W4221055872","https://openalex.org/W4225080353","https://openalex.org/W4225120919","https://openalex.org/W4225165463","https://openalex.org/W4301393026","https://openalex.org/W4307475428","https://openalex.org/W4321161602","https://openalex.org/W4321472415","https://openalex.org/W4361289889","https://openalex.org/W4365601419","https://openalex.org/W4366548330","https://openalex.org/W4366548479","https://openalex.org/W4366729084","https://openalex.org/W4383679928","https://openalex.org/W4383682679","https://openalex.org/W4387801187","https://openalex.org/W4387801427","https://openalex.org/W4389520756","https://openalex.org/W4389988885"],"related_works":["https://openalex.org/W4383501580","https://openalex.org/W4214931137","https://openalex.org/W4387007686","https://openalex.org/W4313813117","https://openalex.org/W4382052417","https://openalex.org/W3192088754","https://openalex.org/W4390193239","https://openalex.org/W3084631705","https://openalex.org/W2388519628","https://openalex.org/W2894473080"],"abstract_inverted_index":{"Large":[0],"language":[1],"model":[2,65,243],"(LLM)":[3],"prompting":[4],"is":[5,168],"a":[6,24,77,82,90,165,175],"promising":[7],"new":[8],"approach":[9],"for":[10,22,105,128],"users":[11,36,61,97,140,189,230],"to":[12,46,59,62,134,184,233,241],"create":[13],"and":[14,31,108,216,246,252],"customize":[15],"their":[16,39,68,74,101,191,199,212,238],"own":[17,192],"chatbots.":[18,137],"However,":[19],"current":[20],"methods":[21],"steering":[23],"chatbot\u2019s":[25,157,172],"outputs,":[26],"such":[27],"as":[28],"prompt":[29,50],"engineering":[30],"fine-tuning,":[32],"do":[33],"not":[34],"support":[35,99,262],"in":[37,48,148],"converting":[38,100,129],"natural":[40,149],"feedback":[41,75,102,131,147,162,213,249],"on":[42],"the":[43,49,86,106,111,156,171,204,235,242,263],"model\u2019s":[44,87],"outputs":[45,66],"changes":[47],"or":[51,145,154],"model.":[52],"In":[53,174],"this":[54,248],"work,":[55],"we":[56,93,122,181],"explore":[57],"how":[58],"enable":[60],"interactively":[63],"refine":[64],"through":[67],"feedback,":[69,153,245],"by":[70,116,119],"helping":[71],"them":[72],"convert":[73,211,247],"into":[76,103,132,170,214,244,250],"set":[78],"of":[79,161,266],"principles":[80,104,200,221],"(i.e.":[81],"constitution)":[83],"that":[84,96,167,198,206,217,261],"dictate":[85],"behavior.":[88],"From":[89],"formative":[91],"study,":[92],"(1)":[94],"found":[95],"needed":[98],"chatbot":[107],"(2)":[109],"classified":[110],"different":[112],"principle":[113,166],"types":[114],"desired":[115],"users.":[117],"Inspired":[118],"these":[120,256],"findings,":[121],"developed":[123],"ConstitutionMaker,":[124,139,195],"an":[125,185],"interactive":[126,264],"tool":[127],"user":[130,176],"principles,":[133,215],"steer":[135],"LLM-based":[136],"With":[138,194],"can":[141],"provide":[142],"either":[143],"positive":[144],"negative":[146],"language,":[150],"select":[151],"auto-generated":[152],"rewrite":[155],"response;":[158],"each":[159],"mode":[160],"automatically":[163],"generates":[164],"inserted":[169],"prompt.":[173],"study":[177],"with":[178,224],"14":[179],"participants,":[180],"compare":[182],"ConstitutionMaker":[183,228],"ablated":[186],"version,":[187],"where":[188],"write":[190,220],"principles.":[193,254],"participants":[196],"felt":[197],"could":[201,208,219],"better":[202],"guide":[203],"chatbot,":[205,236],"they":[207,218],"more":[209,222],"easily":[210],"efficiently,":[223],"less":[225],"mental":[226],"demand.":[227],"helped":[229],"identify":[231],"ways":[232],"improve":[234],"formulate":[237],"intuitive":[239],"responses":[240],"specific":[251],"clear":[253],"Together,":[255],"findings":[257],"inform":[258],"future":[259],"tools":[260],"critiquing":[265],"LLM":[267],"outputs.":[268]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":4}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}