{"id":"https://openalex.org/W3167758559","doi":"https://doi.org/10.1145/3447548.3467086","title":"Real Negatives Matter: Continuous Training with Real Negatives for Delayed Feedback Modeling","display_name":"Real Negatives Matter: Continuous Training with Real Negatives for Delayed Feedback Modeling","publication_year":2021,"publication_date":"2021-08-12","ids":{"openalex":"https://openalex.org/W3167758559","doi":"https://doi.org/10.1145/3447548.3467086","mag":"3167758559"},"language":"en","primary_location":{"id":"doi:10.1145/3447548.3467086","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3447548.3467086","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033003118","display_name":"Siyu Gu","orcid":null},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Siyu Gu","raw_affiliation_strings":["Alibaba Group, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Alibaba Group, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001276052","display_name":"Xiang-Rong Sheng","orcid":"https://orcid.org/0009-0006-4864-574X"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiang-Rong Sheng","raw_affiliation_strings":["Alibaba Group, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Alibaba Group, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100701973","display_name":"Ying Fan","orcid":"https://orcid.org/0000-0002-2368-2572"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ying Fan","raw_affiliation_strings":["Alibaba Group, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Alibaba Group, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108693980","display_name":"Guorui Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guorui Zhou","raw_affiliation_strings":["Alibaba Group, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Alibaba Group, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102812997","display_name":"Xiaoqiang Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoqiang Zhu","raw_affiliation_strings":["Alibaba Group, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Alibaba Group, Beijing, China","institution_ids":["https://openalex.org/I45928872"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5033003118"],"corresponding_institution_ids":["https://openalex.org/I45928872"],"apc_list":null,"apc_paid":null,"fwci":1.6507,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.85841008,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2890","last_page":"2898"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9824000000953674,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11447","display_name":"Blind Source Separation Techniques","score":0.982200026512146,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/false-positive-paradox","display_name":"False positive paradox","score":0.7233905792236328},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7170884609222412},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.7029268741607666},{"id":"https://openalex.org/keywords/false-positives-and-false-negatives","display_name":"False positives and false negatives","score":0.6499959230422974},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5080969333648682},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.382159024477005},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37607359886169434}],"concepts":[{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.7233905792236328},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7170884609222412},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.7029268741607666},{"id":"https://openalex.org/C112789634","wikidata":"https://www.wikidata.org/wiki/Q18207010","display_name":"False positives and false negatives","level":3,"score":0.6499959230422974},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5080969333648682},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.382159024477005},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37607359886169434},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3447548.3467086","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3447548.3467086","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4099999964237213,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W1970210633","https://openalex.org/W2018049374","https://openalex.org/W2021866613","https://openalex.org/W2076618162","https://openalex.org/W2086079772","https://openalex.org/W2088640816","https://openalex.org/W2149822245","https://openalex.org/W2158698691","https://openalex.org/W2171033594","https://openalex.org/W2513944453","https://openalex.org/W2530299800","https://openalex.org/W2604738573","https://openalex.org/W2607662938","https://openalex.org/W2723293840","https://openalex.org/W2913668833","https://openalex.org/W2962745591","https://openalex.org/W2962989965","https://openalex.org/W2973198305","https://openalex.org/W3004726539","https://openalex.org/W3009804075","https://openalex.org/W3100085543","https://openalex.org/W3123656117","https://openalex.org/W3209943551","https://openalex.org/W4248949446"],"related_works":["https://openalex.org/W1557094818","https://openalex.org/W2183246718","https://openalex.org/W1973412793","https://openalex.org/W2099261052","https://openalex.org/W4292605373","https://openalex.org/W2951146195","https://openalex.org/W4226316650","https://openalex.org/W3123215897","https://openalex.org/W2153600354","https://openalex.org/W4243739114"],"abstract_inverted_index":{"One":[0],"of":[1,4,127,133,161,194,214,222,247,258],"the":[2,11,20,43,49,72,79,90,106,111,114,125,131,134,140,143,159,189,198,205,210,223,227,236,245,254],"difficulties":[3],"conversion":[5,73,76,135],"rate":[6],"(CVR)":[7],"prediction":[8],"is":[9,202],"that":[10],"conversions":[12,148],"can":[13],"delay":[14],"and":[15,56,87,271],"take":[16],"place":[17],"long":[18],"after":[19],"clicks.":[21],"The":[22,181,192,212,269],"delayed":[23,162],"feedback":[24],"poses":[25],"a":[26,62,94],"challenge:":[27],"fresh":[28],"data":[29,272],"are":[30,46,145,149,276],"beneficial":[31],"to":[32,124,177,204,234],"continuous":[33],"training":[34,50,91,190],"but":[35],"may":[36],"not":[37,69,121],"have":[38,100,250],"complete":[39],"label":[40,57],"information":[41,221],"at":[42],"time":[44],"they":[45,104],"ingested":[47,88],"into":[48,89,188],"pipeline.":[51,191],"To":[52,225],"balance":[53],"model":[54],"freshness":[55],"certainty,":[58],"previous":[59],"methods":[60,99],"set":[61],"short":[63],"waiting":[64,80],"window":[65],"or":[66],"even":[67],"do":[68],"wait":[70],"for":[71],"signal.":[74],"If":[75],"happens":[77],"outside":[78],"window,":[81],"this":[82,118,165,274],"sample":[83],"will":[84],"be":[85],"duplicated":[86,128],"pipeline":[92],"with":[93,172],"positive":[95],"label.":[96],"However,":[97],"these":[98,179],"some":[101],"issues.":[102,180],"First,":[103],"assume":[105],"observed":[107,199],"feature":[108,200],"distribution":[109,201,228],"remains":[110],"same":[112],"as":[113,147],"actual":[115,206],"distribution.":[116],"But":[117,142],"assumption":[119],"does":[120],"hold":[122],"due":[123],"ingestion":[126,193,213],"samples.":[129],"Second,":[130],"certainty":[132,220],"action":[136],"only":[137],"comes":[138],"from":[139],"positives.":[141],"positives":[144],"scarce":[146],"sparse":[150],"in":[151,253,266,273],"commercial":[152],"systems.":[153],"These":[154],"issues":[155],"induce":[156],"bias":[157],"during":[158],"modeling":[160,171],"feedback.":[163],"In":[164],"paper,":[166],"we":[167],"propose":[168],"DElayed":[169],"FEedback":[170],"Real":[173],"negatives":[174,196,216],"(DEFER)":[175],"method":[176,183],"address":[178],"proposed":[182],"ingests":[184],"real":[185,195,215],"negative":[186],"samples":[187],"ensures":[197],"equivalent":[203],"distribution,":[207],"thus":[208],"reducing":[209],"bias.":[211],"also":[217],"brings":[218],"more":[219],"conversion.":[224],"correct":[226],"shift,":[229],"DEFER":[230,249],"employs":[231],"importance":[232],"sampling":[233],"weigh":[235],"loss":[237],"function.":[238],"Experimental":[239],"results":[240],"on":[241,264],"industrial":[242],"datasets":[243],"validate":[244],"superiority":[246],"DEFER.":[248],"been":[251],"deployed":[252],"display":[255],"advertising":[256],"system":[257],"Alibaba,":[259],"obtaining":[260],"over":[261],"6.0%":[262],"improvement":[263],"CVR":[265],"several":[267],"scenarios.":[268],"code":[270],"paper":[275],"now":[277],"open-sourced":[278],"(https://github.com/gusuperstar/defer.git).":[279]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":5}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}