{"id":"https://openalex.org/W4415324918","doi":"https://doi.org/10.1007/978-3-032-08333-3_3","title":"A Human-in-the-Loop Approach to Learning Social Norms and Behavioural Policies","display_name":"A Human-in-the-Loop Approach to Learning Social Norms and Behavioural Policies","publication_year":2025,"publication_date":"2025-10-18","ids":{"openalex":"https://openalex.org/W4415324918","doi":"https://doi.org/10.1007/978-3-032-08333-3_3"},"language":"en","primary_location":{"id":"doi:10.1007/978-3-032-08333-3_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-08333-3_3","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-032-08333-3_3.pdf","source":{"id":"https://openalex.org/S2764900261","display_name":"Communications in computer and information science","issn_l":"1865-0929","issn":["1865-0929","1865-0937"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications in Computer and Information Science","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/978-3-032-08333-3_3.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009512860","display_name":"Oliver Deane","orcid":"https://orcid.org/0000-0002-9641-4157"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Oliver Deane","raw_affiliation_strings":["University of Bristol, Bristol, BS8 1QU, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-9641-4157","affiliations":[{"raw_affiliation_string":"University of Bristol, Bristol, BS8 1QU, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051924120","display_name":"Oliver Ray","orcid":"https://orcid.org/0000-0002-0221-1501"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Oliver Ray","raw_affiliation_strings":["University of Bristol, Bristol, BS8 1QU, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0002-0221-1501","affiliations":[{"raw_affiliation_string":"University of Bristol, Bristol, BS8 1QU, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5009512860"],"corresponding_institution_ids":["https://openalex.org/I36234482"],"apc_list":null,"apc_paid":null,"fwci":4.1459,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.94953096,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"41","last_page":"63"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11810","display_name":"Complex Systems and Decision Making","score":0.9387999773025513,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/normative","display_name":"Normative","score":0.7235000133514404},{"id":"https://openalex.org/keywords/imitation","display_name":"Imitation","score":0.5522000193595886},{"id":"https://openalex.org/keywords/formalism","display_name":"Formalism (music)","score":0.5289999842643738},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5166000127792358},{"id":"https://openalex.org/keywords/constraint","display_name":"Constraint (computer-aided design)","score":0.3808000087738037},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.3693000078201294},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.3596000075340271},{"id":"https://openalex.org/keywords/competence","display_name":"Competence (human resources)","score":0.34529998898506165}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8021000027656555},{"id":"https://openalex.org/C44725695","wikidata":"https://www.wikidata.org/wiki/Q288156","display_name":"Normative","level":2,"score":0.7235000133514404},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.5522000193595886},{"id":"https://openalex.org/C73301696","wikidata":"https://www.wikidata.org/wiki/Q5469984","display_name":"Formalism (music)","level":3,"score":0.5289999842643738},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5166000127792358},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4722999930381775},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.3808000087738037},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.3693000078201294},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.34529998898506165},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.33399999141693115},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3228999972343445},{"id":"https://openalex.org/C79416737","wikidata":"https://www.wikidata.org/wiki/Q2305519","display_name":"Social learning","level":2,"score":0.29820001125335693},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.2890999913215637},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.28769999742507935},{"id":"https://openalex.org/C21847791","wikidata":"https://www.wikidata.org/wiki/Q191081","display_name":"Logical conjunction","level":2,"score":0.2728999853134155},{"id":"https://openalex.org/C171018156","wikidata":"https://www.wikidata.org/wiki/Q7370306","display_name":"Rotation formalisms in three dimensions","level":2,"score":0.2671999931335449},{"id":"https://openalex.org/C43971567","wikidata":"https://www.wikidata.org/wiki/Q3142865","display_name":"Logical reasoning","level":2,"score":0.2574000060558319},{"id":"https://openalex.org/C137549413","wikidata":"https://www.wikidata.org/wiki/Q7053127","display_name":"Normative model of decision-making","level":3,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/978-3-032-08333-3_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-08333-3_3","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-032-08333-3_3.pdf","source":{"id":"https://openalex.org/S2764900261","display_name":"Communications in computer and information science","issn_l":"1865-0929","issn":["1865-0929","1865-0937"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications in Computer and Information Science","raw_type":"book-chapter"}],"best_oa_location":{"id":"doi:10.1007/978-3-032-08333-3_3","is_oa":true,"landing_page_url":"https://doi.org/10.1007/978-3-032-08333-3_3","pdf_url":"https://link.springer.com/content/pdf/10.1007/978-3-032-08333-3_3.pdf","source":{"id":"https://openalex.org/S2764900261","display_name":"Communications in computer and information science","issn_l":"1865-0929","issn":["1865-0929","1865-0937"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications in Computer and Information Science","raw_type":"book-chapter"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415324918.pdf","grobid_xml":"https://content.openalex.org/works/W4415324918.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W593005562","https://openalex.org/W1552549180","https://openalex.org/W1572717432","https://openalex.org/W1864247448","https://openalex.org/W1890330145","https://openalex.org/W2009605580","https://openalex.org/W2023808162","https://openalex.org/W2039868847","https://openalex.org/W2107726111","https://openalex.org/W2130304665","https://openalex.org/W2340952989","https://openalex.org/W2766447205","https://openalex.org/W2963575966","https://openalex.org/W2964627913","https://openalex.org/W2973186106","https://openalex.org/W3065734471","https://openalex.org/W3122928565","https://openalex.org/W3127561923","https://openalex.org/W3130163632","https://openalex.org/W3178674171","https://openalex.org/W4200039633","https://openalex.org/W4324131393","https://openalex.org/W4360978713","https://openalex.org/W4392964192","https://openalex.org/W4407831732"],"related_works":[],"abstract_inverted_index":{"Abstract":[0],"It":[1],"is":[2,41,75],"often":[3],"desirable":[4],"to":[5,11,21,30,46,50,61,78,152,158,220,247,260,275],"constrain":[6],"reinforcement":[7],"learning":[8,261],"(RL)":[9],"policies":[10],"align":[12],"with":[13,33,98,125,146,283],"societal":[14],"norms":[15,91,196],"and":[16,26,35,49,102,167,181,200,240,249,268,279],"individual":[17],"preferences":[18],"in":[19,66,81,206,281],"order":[20,29],"better":[22],"represent":[23],"users\u2019":[24],"intentions":[25],"expectations.":[27],"In":[28],"adequately":[31],"deal":[32],"exceptions":[34],"conflicts":[36],"between":[37],"competing":[38],"norms/preferences,":[39],"it":[40,74,244],"useful":[42],"for":[43,54,178,189,265],"such":[44],"constraints":[45,120,144,215,264,274],"be":[47,62,93,276],"defeasible;":[48],"enable":[51],"transparent":[52],"justification":[53],"decisions,":[55],"as":[56,58],"well":[57],"allowing":[59,156],"them":[60,80],"more":[63],"easily":[64],"re-used":[65],"different":[67],"variations":[68],"of":[69,105,132,141,233],"a":[70,82,99,114,139,207,257],"particular":[71],"problem":[72],"setting,":[73],"also":[76],"helpful":[77],"express":[79],"high-level,":[83],"symbolic,":[84,262],"human-understandable":[85],"form.":[86],"However,":[87],"manually":[88],"specifying":[89],"these":[90],"can":[92],"challenging,":[94],"typically":[95],"requiring":[96,224],"familiarity":[97],"logical":[100,143,214],"formalism":[101],"extensive":[103],"knowledge":[104],"the":[106,133,212,231],"application":[107],"domain.":[108],"To":[109],"address":[110],"this,":[111],"we":[112,229],"propose":[113],"human-in-the-loop":[115],"approach":[116,259],"that":[117,150,211,243,272],"learns":[118],"ethical":[119,165],"by":[121],"combining":[122],"autonomous":[123],"exploration":[124,180],"expert":[126],"imitation":[127],"from":[128],"simple":[129],"user-provided":[130],"examples":[131],"intended":[134],"behaviour.":[135],"Our":[136],"system":[137,171],"infers":[138],"list":[140],"high-level":[142],"along":[145],"an":[147,173,182],"RL":[148,266],"policy":[149],"adheres":[151],"them,":[153],"while":[154],"optionally":[155],"users":[157],"interactively":[159],"refine":[160],"constraints,":[161],"resolve":[162],"conflicts,":[163],"manage":[164],"contradictions":[166],"provide":[168],"counter-examples.":[169],"The":[170],"interleaves":[172],"Approximate":[174],"Q-learning":[175],"(AQL)":[176],"component":[177],"goal-directed":[179],"interactive":[183,237,270],"Inductive":[184],"Logic":[185],"Programming":[186],"(ILP)":[187],"module":[188],"symbolic":[190],"constraint":[191],"inference,":[192],"thus":[193],"ensuring":[194],"learned":[195,213],"remain":[197],"transparent,":[198],"auditable,":[199],"editable.":[201],"We":[202],"evaluate":[203],"our":[204],"method":[205],"Pacman":[208],"environment,":[209],"demonstrating":[210],"achieve":[216],"normative":[217,251],"compliance":[218],"comparable":[219],"existing":[221],"approaches,":[222],"without":[223],"manual":[225],"rule":[226],"specification.":[227],"Furthermore,":[228],"demonstrate":[230],"value":[232],"human":[234],"intervention":[235],"via":[236],"ILP":[238],"mechanisms,":[239],"experimentally":[241],"show":[242],"accelerates":[245],"convergence":[246],"accurate":[248],"efficient":[250],"frameworks.":[252],"This":[253],"work":[254],"therefore":[255],"contributes":[256],"novel":[258],"defeasible":[263],"policies,":[267],"introduces":[269],"mechanisms":[271],"allow":[273],"tailored,":[277],"refined,":[278],"overridden":[280],"accordance":[282],"user":[284],"intentions.":[285]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-14T06:41:57.775601","created_date":"2025-10-19T00:00:00"}