{"id":"https://openalex.org/W2988177603","doi":"https://doi.org/10.1017/s0269888919000092","title":"Two-level Q-learning: learning from conflict demonstrations","display_name":"Two-level Q-learning: learning from conflict demonstrations","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2988177603","doi":"https://doi.org/10.1017/s0269888919000092","mag":"2988177603"},"language":"en","primary_location":{"id":"doi:10.1017/s0269888919000092","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s0269888919000092","pdf_url":null,"source":{"id":"https://openalex.org/S137506714","display_name":"The Knowledge Engineering Review","issn_l":"0269-8889","issn":["0269-8889","1469-8005"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Knowledge Engineering Review","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100355195","display_name":"Mao Li","orcid":"https://orcid.org/0000-0001-8500-1165"},"institutions":[{"id":"https://openalex.org/I52099693","display_name":"University of York","ror":"https://ror.org/04m01e293","country_code":"GB","type":"education","lineage":["https://openalex.org/I52099693"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Mao Li","raw_affiliation_strings":["Computer Science Department, University of York, York, United Kingdom e-mail:","University of York"],"raw_orcid":"https://orcid.org/0000-0001-8500-1165","affiliations":[{"raw_affiliation_string":"Computer Science Department, University of York, York, United Kingdom e-mail:","institution_ids":["https://openalex.org/I52099693"]},{"raw_affiliation_string":"University of York","institution_ids":["https://openalex.org/I52099693"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100339151","display_name":"Wei Yi","orcid":"https://orcid.org/0000-0001-9878-7048"},"institutions":[{"id":"https://openalex.org/I34949971","display_name":"University of Jinan","ror":"https://ror.org/02mjz6f26","country_code":"CN","type":"education","lineage":["https://openalex.org/I34949971"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yi Wei","raw_affiliation_strings":["Computer Science Department, Shan dong University, Jinan, China e-mail:","Shan dong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department, Shan dong University, Jinan, China e-mail:","institution_ids":["https://openalex.org/I34949971"]},{"raw_affiliation_string":"Shan dong University","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5009587907","display_name":"Daniel Kudenko\u22c6","orcid":"https://orcid.org/0000-0003-3359-3255"},"institutions":[{"id":"https://openalex.org/I2895403630","display_name":"St. Petersburg Institute of Jewish Studies","ror":"https://ror.org/00qqptx17","country_code":"RU","type":"education","lineage":["https://openalex.org/I2895403630"]},{"id":"https://openalex.org/I52099693","display_name":"University of York","ror":"https://ror.org/04m01e293","country_code":"GB","type":"education","lineage":["https://openalex.org/I52099693"]}],"countries":["GB","RU"],"is_corresponding":false,"raw_author_name":"Daniel Kudenko","raw_affiliation_strings":["Computer Science Department, University of York, York, United Kingdom e-mail:","JetBrains Research, St Petersburg, Russia e-mail:","University of York"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department, University of York, York, United Kingdom e-mail:","institution_ids":["https://openalex.org/I52099693"]},{"raw_affiliation_string":"JetBrains Research, St Petersburg, Russia e-mail:","institution_ids":["https://openalex.org/I2895403630"]},{"raw_affiliation_string":"University of York","institution_ids":["https://openalex.org/I52099693"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5100355195"],"corresponding_institution_ids":["https://openalex.org/I52099693"],"apc_list":null,"apc_paid":null,"fwci":0.4236,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.72016454,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"34","issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9686999917030334,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.9678000211715698,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8080830574035645},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7739665508270264},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6586400866508484},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.6470404267311096},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6192423701286316},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.5755330324172974},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.574510395526886},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5051787495613098},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.47506850957870483},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.43502938747406006},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.41301536560058594}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8080830574035645},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7739665508270264},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6586400866508484},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.6470404267311096},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6192423701286316},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.5755330324172974},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.574510395526886},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5051787495613098},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.47506850957870483},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.43502938747406006},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.41301536560058594},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1017/s0269888919000092","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s0269888919000092","pdf_url":null,"source":{"id":"https://openalex.org/S137506714","display_name":"The Knowledge Engineering Review","issn_l":"0269-8889","issn":["0269-8889","1469-8005"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Knowledge Engineering Review","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6700000166893005,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1507087299","https://openalex.org/W1777239053","https://openalex.org/W2031727428","https://openalex.org/W2121517924","https://openalex.org/W2121863487","https://openalex.org/W2137375617","https://openalex.org/W2145339207","https://openalex.org/W2148112459","https://openalex.org/W2151382427","https://openalex.org/W2164419340","https://openalex.org/W2397581010","https://openalex.org/W2740302738","https://openalex.org/W2788862220","https://openalex.org/W2963376229","https://openalex.org/W4214717370","https://openalex.org/W6682205418"],"related_works":["https://openalex.org/W2383111961","https://openalex.org/W2365952365","https://openalex.org/W2352448290","https://openalex.org/W2380820513","https://openalex.org/W2005801347","https://openalex.org/W2372385138","https://openalex.org/W2186640231","https://openalex.org/W4296359239","https://openalex.org/W1557905920","https://openalex.org/W2043093291"],"abstract_inverted_index":{"Abstract":[0],"One":[1],"way":[2],"to":[3,14,19,48,113,120,160,165,194],"address":[4],"this":[5],"low":[6],"sample":[7],"efficiency":[8],"of":[9,65,84,104,173],"reinforcement":[10],"learning":[11,185],"(RL)":[12],"is":[13,72],"employ":[15],"human":[16],"expert":[17,118],"demonstrations":[18,37,52,76,132],"speed":[20],"up":[21],"the":[22,49,66,75,85,96,102,107,115,121,128,151,155,170,184,188,192],"RL":[23,97],"process":[24,186],"(RL":[25],"from":[26,38,55,134],"demonstration":[27,196],"or":[28,162],"RLfD).":[29],"The":[30],"research":[31],"so":[32],"far":[33],"has":[34,45,191],"focused":[35],"on":[36,62,106,146],"a":[39,90],"single":[40,136],"expert.":[41],"However,":[42],"little":[43],"attention":[44],"been":[46],"given":[47],"case":[50],"where":[51],"are":[53,139],"collected":[54],"multiple":[56],"experts,":[57],"whose":[58],"expertise":[59],"may":[60],"vary":[61],"different":[63,148],"aspects":[64],"task.":[67],"In":[68,168],"such":[69],"scenarios,":[70],"it":[71],"likely":[73],"that":[74,131,154],"will":[77],"contain":[78],"conflicting":[79],"advice":[80],"in":[81,94,183],"many":[82],"parts":[83],"state":[86],"space.":[87],"We":[88,142],"propose":[89],"two-level":[91],"Q-learning":[92],"algorithm,":[93],"which":[95],"agent":[98],"not":[99],"only":[100],"learns":[101,112],"policy":[103],"deciding":[105],"optimal":[108],"action":[109],"but":[110],"also":[111],"select":[114],"most":[116],"trustworthy":[117],"according":[119],"current":[122],"state.":[123],"Thus,":[124],"our":[125,144,174],"approach":[126,190],"removes":[127],"traditional":[129],"assumption":[130],"come":[133],"one":[135],"source":[137],"and":[138,150,187],"mostly":[140],"conflict-free.":[141],"evaluate":[143],"technique":[145],"three":[147],"domains":[149],"results":[152],"show":[153],"state-of-the-art":[156],"RLfD":[157],"baseline":[158],"fails":[159],"converge":[161],"performs":[163],"similarly":[164],"conventional":[166],"Q-learning.":[167],"contrast,":[169],"performance":[171],"level":[172],"novel":[175],"algorithm":[176],"increases":[177],"with":[178],"more":[179],"experts":[180],"being":[181],"involved":[182],"proposed":[189],"capability":[193],"handle":[195],"conflicts":[197],"well.":[198]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}