{"id":"https://openalex.org/W2741687119","doi":"https://doi.org/10.1145/3077136.3080808","title":"Online In-Situ Interleaved Evaluation of Real-Time Push Notification Systems","display_name":"Online In-Situ Interleaved Evaluation of Real-Time Push Notification Systems","publication_year":2017,"publication_date":"2017-07-28","ids":{"openalex":"https://openalex.org/W2741687119","doi":"https://doi.org/10.1145/3077136.3080808","mag":"2741687119"},"language":"en","primary_location":{"id":"doi:10.1145/3077136.3080808","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3080808","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048499958","display_name":"Adam Roegiest","orcid":"https://orcid.org/0000-0003-1265-8881"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Adam Roegiest","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043679408","display_name":"Luchen Tan","orcid":null},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Luchen Tan","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5048499958"],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":7.5883,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.97048292,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"415","last_page":"424"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10064","display_name":"Complex Network Analysis Techniques","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.9133925437927246},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8456844091415405},{"id":"https://openalex.org/keywords/push-technology","display_name":"Push technology","score":0.7182360887527466},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.6828963756561279},{"id":"https://openalex.org/keywords/notification-system","display_name":"Notification system","score":0.6674954891204834},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5569359660148621},{"id":"https://openalex.org/keywords/mobile-device","display_name":"Mobile device","score":0.521450936794281},{"id":"https://openalex.org/keywords/social-media","display_name":"Social media","score":0.4469894766807556},{"id":"https://openalex.org/keywords/track","display_name":"Track (disk drive)","score":0.43727123737335205},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.34647154808044434},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.26005202531814575},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.08686581254005432}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9133925437927246},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8456844091415405},{"id":"https://openalex.org/C180652500","wikidata":"https://www.wikidata.org/wiki/Q1351910","display_name":"Push technology","level":2,"score":0.7182360887527466},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.6828963756561279},{"id":"https://openalex.org/C2779011373","wikidata":"https://www.wikidata.org/wiki/Q3962191","display_name":"Notification system","level":2,"score":0.6674954891204834},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5569359660148621},{"id":"https://openalex.org/C186967261","wikidata":"https://www.wikidata.org/wiki/Q5082128","display_name":"Mobile device","level":2,"score":0.521450936794281},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.4469894766807556},{"id":"https://openalex.org/C89992363","wikidata":"https://www.wikidata.org/wiki/Q5961558","display_name":"Track (disk drive)","level":2,"score":0.43727123737335205},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.34647154808044434},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26005202531814575},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.08686581254005432},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3077136.3080808","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3077136.3080808","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W18860330","https://openalex.org/W1594112393","https://openalex.org/W1798038286","https://openalex.org/W1838260653","https://openalex.org/W1977389525","https://openalex.org/W1980036597","https://openalex.org/W1982530130","https://openalex.org/W1998955392","https://openalex.org/W2006811637","https://openalex.org/W2008285050","https://openalex.org/W2037377096","https://openalex.org/W2069003154","https://openalex.org/W2090715008","https://openalex.org/W2106365165","https://openalex.org/W2116008435","https://openalex.org/W2135046706","https://openalex.org/W2153256230","https://openalex.org/W2156186232","https://openalex.org/W2156541733","https://openalex.org/W2160471965","https://openalex.org/W2163996478","https://openalex.org/W2167510893","https://openalex.org/W2169783907","https://openalex.org/W2239566685","https://openalex.org/W2271209846","https://openalex.org/W2338920854","https://openalex.org/W2399079233","https://openalex.org/W2406393172","https://openalex.org/W2467224519","https://openalex.org/W2643544958","https://openalex.org/W2915979700","https://openalex.org/W2918757710","https://openalex.org/W3012992600"],"related_works":["https://openalex.org/W2234151072","https://openalex.org/W2588119133","https://openalex.org/W1481484839","https://openalex.org/W2911420370","https://openalex.org/W4304633963","https://openalex.org/W4378976719","https://openalex.org/W4251469996","https://openalex.org/W1509996184","https://openalex.org/W2936630017","https://openalex.org/W2783007838"],"abstract_inverted_index":{"Real-time":[0],"push":[1,49],"notification":[2],"systems":[3,31],"monitor":[4],"continuous":[5],"document":[6],"streams":[7],"such":[8,30],"as":[9,48,151],"social":[10],"media":[11],"posts":[12],"and":[13,78,115],"alert":[14],"users":[15],"to":[16,51,63,83,141],"relevant":[17],"content":[18],"directly":[19],"on":[20],"their":[21],"mobile":[22,53],"devices.":[23],"We":[24,108],"describe":[25],"a":[26,56,88,102,148],"user":[27,85],"study":[28,61],"of":[29,35,55,58,69,92,145,155],"in":[32,87],"the":[33,36,52,66,100,124,143],"context":[34],"TREC":[37],"2016":[38],"Real-Time":[39],"Summarization":[40],"Track,":[41],"where":[42],"system":[43,149],"updates":[44],"are":[45,97,129,139],"immediately":[46],"delivered":[47],"notifications":[50],"devices":[54],"cohort":[57],"users.":[59],"Our":[60],"represents,":[62],"our":[64,93],"knowledge,":[65],"first":[67],"deployment":[68],"an":[70,81],"interleaved":[71],"evaluation":[72,96,117],"framework":[73],"for":[74,120],"prospective":[75],"information":[76],"needs,":[77],"also":[79],"provides":[80],"opportunity":[82],"examine":[84],"behavior":[86],"realistic":[89],"setting.":[90],"Results":[91],"online":[94,114],"in-situ":[95],"correlated":[98],"against":[99],"results":[101],"more":[103],"traditional":[104],"post-hoc":[105],"batch":[106,116],"evaluation.":[107],"observe":[109,135],"substantial":[110],"correlations":[111],"between":[112],"many":[113],"metrics,":[118,133],"especially":[119],"those":[121],"that":[122,147],"share":[123],"same":[125],"basic":[126],"design":[127],"(e.g.,":[128],"utility-based).":[130],"For":[131],"some":[132],"we":[134],"little":[136],"correlation,":[137],"but":[138],"able":[140],"identify":[142],"volume":[144],"messages":[146],"pushes":[150],"one":[152],"major":[153],"source":[154],"differences.":[156]},"counts_by_year":[{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":5}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}