{"id":"https://openalex.org/W3156554472","doi":"https://doi.org/10.1007/s10994-021-05969-w","title":"Partially observable environment estimation with uplift inference for reinforcement learning based recommendation","display_name":"Partially observable environment estimation with uplift inference for reinforcement learning based recommendation","publication_year":2021,"publication_date":"2021-04-14","ids":{"openalex":"https://openalex.org/W3156554472","doi":"https://doi.org/10.1007/s10994-021-05969-w","mag":"3156554472"},"language":"en","primary_location":{"id":"doi:10.1007/s10994-021-05969-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10994-021-05969-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10994-021-05969-w.pdf","source":{"id":"https://openalex.org/S62148650","display_name":"Machine Learning","issn_l":"0885-6125","issn":["0885-6125","1573-0565"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10994-021-05969-w.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002985691","display_name":"Wenjie Shang","orcid":"https://orcid.org/0000-0002-9331-4062"},"institutions":[{"id":"https://openalex.org/I4401726870","display_name":"Didi Chuxing (China)","ror":"https://ror.org/02ksqcf75","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726870"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Wenjie Shang","raw_affiliation_strings":["AI Labs, Didi Chuxing, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9331-4062","affiliations":[{"raw_affiliation_string":"AI Labs, Didi Chuxing, Beijing, China","institution_ids":["https://openalex.org/I4401726870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100700129","display_name":"Qingyang Li","orcid":"https://orcid.org/0000-0001-6650-2343"},"institutions":[{"id":"https://openalex.org/I4401726870","display_name":"Didi Chuxing (China)","ror":"https://ror.org/02ksqcf75","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726870"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingyang Li","raw_affiliation_strings":["AI Labs, Didi Chuxing, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AI Labs, Didi Chuxing, Beijing, China","institution_ids":["https://openalex.org/I4401726870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085579946","display_name":"Zhiwei Qin","orcid":"https://orcid.org/0000-0001-5383-4816"},"institutions":[{"id":"https://openalex.org/I4401726870","display_name":"Didi Chuxing (China)","ror":"https://ror.org/02ksqcf75","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726870"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiwei Qin","raw_affiliation_strings":["AI Labs, Didi Chuxing, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AI Labs, Didi Chuxing, Beijing, China","institution_ids":["https://openalex.org/I4401726870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100342263","display_name":"Yang Yu","orcid":"https://orcid.org/0000-0002-1732-9545"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang Yu","raw_affiliation_strings":["National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, 210023, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, 210023, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112633752","display_name":"Yiping Meng","orcid":null},"institutions":[{"id":"https://openalex.org/I4401726870","display_name":"Didi Chuxing (China)","ror":"https://ror.org/02ksqcf75","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726870"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiping Meng","raw_affiliation_strings":["AI Labs, Didi Chuxing, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AI Labs, Didi Chuxing, Beijing, China","institution_ids":["https://openalex.org/I4401726870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010419481","display_name":"Jieping Ye","orcid":"https://orcid.org/0000-0001-8662-5818"},"institutions":[{"id":"https://openalex.org/I4401726870","display_name":"Didi Chuxing (China)","ror":"https://ror.org/02ksqcf75","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726870"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jieping Ye","raw_affiliation_strings":["AI Labs, Didi Chuxing, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AI Labs, Didi Chuxing, Beijing, China","institution_ids":["https://openalex.org/I4401726870"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5002985691"],"corresponding_institution_ids":["https://openalex.org/I4401726870"],"apc_list":{"value":2390,"currency":"EUR","value_usd":2990},"apc_paid":{"value":2390,"currency":"EUR","value_usd":2990},"fwci":0.9554,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.79366727,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"110","issue":"9","first_page":"2603","last_page":"2640"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10603","display_name":"Smart Grid Energy Management","score":0.9732999801635742,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7941012978553772},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7453740835189819},{"id":"https://openalex.org/keywords/causal-inference","display_name":"Causal inference","score":0.702424168586731},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6937662363052368},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.621989905834198},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6138759851455688},{"id":"https://openalex.org/keywords/estimation","display_name":"Estimation","score":0.5972700119018555},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.12315458059310913},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10345685482025146}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7941012978553772},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7453740835189819},{"id":"https://openalex.org/C158600405","wikidata":"https://www.wikidata.org/wiki/Q5054566","display_name":"Causal inference","level":2,"score":0.702424168586731},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6937662363052368},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.621989905834198},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6138759851455688},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.5972700119018555},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.12315458059310913},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10345685482025146},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s10994-021-05969-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10994-021-05969-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10994-021-05969-w.pdf","source":{"id":"https://openalex.org/S62148650","display_name":"Machine Learning","issn_l":"0885-6125","issn":["0885-6125","1573-0565"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s10994-021-05969-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10994-021-05969-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10994-021-05969-w.pdf","source":{"id":"https://openalex.org/S62148650","display_name":"Machine Learning","issn_l":"0885-6125","issn":["0885-6125","1573-0565"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Learning","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5438160124","display_name":null,"funder_award_id":"2018AAA0101100","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5469316478","display_name":null,"funder_award_id":"61876077","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3156554472.pdf","grobid_xml":"https://content.openalex.org/works/W3156554472.grobid-xml"},"referenced_works_count":61,"referenced_works":["https://openalex.org/W146900863","https://openalex.org/W162576350","https://openalex.org/W1532688806","https://openalex.org/W1541084404","https://openalex.org/W1570649553","https://openalex.org/W1603952207","https://openalex.org/W1931877416","https://openalex.org/W1974449456","https://openalex.org/W1976459656","https://openalex.org/W1981886486","https://openalex.org/W1986014385","https://openalex.org/W2011418219","https://openalex.org/W2011485768","https://openalex.org/W2031571562","https://openalex.org/W2042206577","https://openalex.org/W2051228319","https://openalex.org/W2065733836","https://openalex.org/W2099471712","https://openalex.org/W2114501936","https://openalex.org/W2121863487","https://openalex.org/W2145339207","https://openalex.org/W2145727188","https://openalex.org/W2148771208","https://openalex.org/W2150291618","https://openalex.org/W2166302491","https://openalex.org/W2168359464","https://openalex.org/W2208550830","https://openalex.org/W2249217595","https://openalex.org/W2257979135","https://openalex.org/W2295598076","https://openalex.org/W2389937032","https://openalex.org/W2559655401","https://openalex.org/W2566467060","https://openalex.org/W2612780607","https://openalex.org/W2614208603","https://openalex.org/W2624816748","https://openalex.org/W2742407816","https://openalex.org/W2802494201","https://openalex.org/W2804132768","https://openalex.org/W2808805866","https://openalex.org/W2902630600","https://openalex.org/W2911964244","https://openalex.org/W2919013397","https://openalex.org/W2949608212","https://openalex.org/W2950946978","https://openalex.org/W2951625652","https://openalex.org/W2962957005","https://openalex.org/W2962957031","https://openalex.org/W2963277051","https://openalex.org/W2964157711","https://openalex.org/W2972535955","https://openalex.org/W2979827629","https://openalex.org/W2996037775","https://openalex.org/W3030981716","https://openalex.org/W3087809036","https://openalex.org/W3102476541","https://openalex.org/W4242356323","https://openalex.org/W4285719527","https://openalex.org/W6638018090","https://openalex.org/W6711702658","https://openalex.org/W6718092244"],"related_works":["https://openalex.org/W4306904969","https://openalex.org/W2138720691","https://openalex.org/W4362501864","https://openalex.org/W4380318855","https://openalex.org/W2031695474","https://openalex.org/W2024136090","https://openalex.org/W2586732548","https://openalex.org/W3049728571","https://openalex.org/W20361778","https://openalex.org/W2964765435"],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"(RL)":[2],"aims":[3],"at":[4],"searching":[5],"the":[6,24,38,42,52,57,67,77,83,88,112,121,128,146,155,173,181,185,209,234,243,257,282],"best":[7],"policy":[8,25,39,280],"model":[9,86,170,183],"for":[10,18],"decision":[11],"making,":[12],"and":[13,160,217],"has":[14],"been":[15],"shown":[16],"powerful":[17],"sequential":[19],"recommendations.":[20],"The":[21,74,267],"training":[22,40],"of":[23,69,76,120,154,176,211,236,246],"by":[26,126],"RL,":[27],"however,":[28],"is":[29,60],"placed":[30],"in":[31,41,71,184,214,242,281],"an":[32,47,62,117,223],"environment.":[33,122,148,266],"In":[34,123],"many":[35],"real-world":[36,91,218,230,283],"applications,":[37],"real":[43,244],"environment":[44,78,101,140,182,201],"can":[45,115,254,275],"cause":[46],"unbearable":[48],"cost":[49],"due":[50],"to":[51,65,81,97,144,171,197,232,261],"exploration.":[53],"Environment":[54],"estimation":[55,75,119,141],"from":[56,87,228],"past":[58],"data":[59],"thus":[61],"appealing":[63],"way":[64],"release":[66],"power":[68],"RL":[70],"these":[72],"applications.":[73],"is,":[79],"basically,":[80],"extract":[82],"causal":[84,156,174,204],"effect":[85,210],"data.":[89],"However,":[90],"applications":[92],"are":[93,107],"often":[94],"too":[95],"complex":[96],"offer":[98],"fully":[99],"observable":[100],"information.":[102],"Therefore,":[103],"quite":[104],"possibly":[105],"there":[106],"unobserved":[108],"variables":[109,130],"lying":[110],"behind":[111],"data,":[113],"which":[114],"obstruct":[116],"effective":[118],"this":[124],"paper,":[125],"treating":[127],"hidden":[129,133,258],"as":[131],"a":[132,137,151,164,190,199,203,229,262,277],"policy,":[134],"we":[135,162,188],"propose":[136,189],"partially-observed":[138,147,200],"multi-agent":[139],"(POMEE)":[142],"approach":[143,196],"learn":[145,172],"To":[149],"make":[150],"better":[152],"extraction":[153],"relationship":[157],"between":[158],"actions":[159],"rewards,":[161],"design":[163],"deep":[165],"uplift":[166,193],"inference":[167,194],"network":[168],"(DUIN)":[169],"effects":[175],"different":[177],"actions.":[178],"By":[179],"implementing":[180],"DUIN":[186],"structure,":[187],"POMEE":[191,274],"with":[192,202],"(POMEE-UI)":[195],"generate":[198],"reward":[205],"mechanism.":[206],"We":[207,220,238],"analyze":[208],"our":[212],"method":[213],"both":[215],"artificial":[216,224],"environments.":[219],"first":[221],"use":[222],"recommender":[225,279],"environment,":[226],"abstracted":[227],"application,":[231],"verify":[233],"effectiveness":[235],"POMEE-UI.":[237],"then":[239],"test":[240],"POMEE-UI":[241,253],"application":[245],"Didi":[247],"Chuxing.":[248],"Experiment":[249],"results":[250,271],"show":[251,272],"that":[252,273],"effectively":[255],"estimate":[256],"variables,":[259],"leading":[260],"more":[263],"reliable":[264],"virtual":[265],"online":[268],"A/B":[269],"testing":[270],"derive":[276],"well-performing":[278],"application.":[284]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}