{"id":"https://openalex.org/W4320024150","doi":"https://doi.org/10.1109/bigdata55660.2022.10020316","title":"Latency vs Cost : Data Ingestion options at Twitter Scale","display_name":"Latency vs Cost : Data Ingestion options at Twitter Scale","publication_year":2022,"publication_date":"2022-12-17","ids":{"openalex":"https://openalex.org/W4320024150","doi":"https://doi.org/10.1109/bigdata55660.2022.10020316"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata55660.2022.10020316","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/bigdata55660.2022.10020316","pdf_url":null,"source":{"id":"https://openalex.org/S4363607709","display_name":"2022 IEEE International Conference on Big Data (Big Data)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060841484","display_name":"Zhenzhao Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I113979032","display_name":"Twitter (United States)","ror":"https://ror.org/04wt43v05","country_code":"US","type":"company","lineage":["https://openalex.org/I113979032"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Zhenzhao Wang","raw_affiliation_strings":["Twitter, Inc"],"affiliations":[{"raw_affiliation_string":"Twitter, Inc","institution_ids":["https://openalex.org/I113979032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030225504","display_name":"Santosh Marella","orcid":null},"institutions":[{"id":"https://openalex.org/I113979032","display_name":"Twitter (United States)","ror":"https://ror.org/04wt43v05","country_code":"US","type":"company","lineage":["https://openalex.org/I113979032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Santosh Marella","raw_affiliation_strings":["Twitter, Inc"],"affiliations":[{"raw_affiliation_string":"Twitter, Inc","institution_ids":["https://openalex.org/I113979032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054262389","display_name":"Mak Inada","orcid":null},"institutions":[{"id":"https://openalex.org/I113979032","display_name":"Twitter (United States)","ror":"https://ror.org/04wt43v05","country_code":"US","type":"company","lineage":["https://openalex.org/I113979032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mak Inada","raw_affiliation_strings":["Twitter, Inc"],"affiliations":[{"raw_affiliation_string":"Twitter, Inc","institution_ids":["https://openalex.org/I113979032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001955786","display_name":"Pablo Rodriguez Defino","orcid":null},"institutions":[{"id":"https://openalex.org/I113979032","display_name":"Twitter (United States)","ror":"https://ror.org/04wt43v05","country_code":"US","type":"company","lineage":["https://openalex.org/I113979032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pablo Rodriguez Defino","raw_affiliation_strings":["Twitter, Inc"],"affiliations":[{"raw_affiliation_string":"Twitter, Inc","institution_ids":["https://openalex.org/I113979032"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056078947","display_name":"Abhishek Jagannath","orcid":null},"institutions":[{"id":"https://openalex.org/I113979032","display_name":"Twitter (United States)","ror":"https://ror.org/04wt43v05","country_code":"US","type":"company","lineage":["https://openalex.org/I113979032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abhishek Jagannath","raw_affiliation_strings":["Twitter, Inc"],"affiliations":[{"raw_affiliation_string":"Twitter, Inc","institution_ids":["https://openalex.org/I113979032"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5009056866","display_name":"Lohit VijayaRenu","orcid":null},"institutions":[{"id":"https://openalex.org/I113979032","display_name":"Twitter (United States)","ror":"https://ror.org/04wt43v05","country_code":"US","type":"company","lineage":["https://openalex.org/I113979032"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lohit VijayaRenu","raw_affiliation_strings":["Twitter, Inc"],"affiliations":[{"raw_affiliation_string":"Twitter, Inc","institution_ids":["https://openalex.org/I113979032"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5060841484"],"corresponding_institution_ids":["https://openalex.org/I113979032"],"apc_list":null,"apc_paid":null,"fwci":0.2574,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.49544724,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"223","last_page":"227"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10317","display_name":"Advanced Database Systems and Queries","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10742","display_name":"Peer-to-Peer Network Technologies","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/petabyte","display_name":"Petabyte","score":0.961564302444458},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7051525115966797},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.573045015335083},{"id":"https://openalex.org/keywords/ingestion","display_name":"Ingestion","score":0.4812157452106476},{"id":"https://openalex.org/keywords/big-data","display_name":"Big data","score":0.41746512055397034},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.39044904708862305},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.3305298686027527},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.19755101203918457},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.15703800320625305}],"concepts":[{"id":"https://openalex.org/C13600138","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Petabyte","level":3,"score":0.961564302444458},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7051525115966797},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.573045015335083},{"id":"https://openalex.org/C193230392","wikidata":"https://www.wikidata.org/wiki/Q1663054","display_name":"Ingestion","level":2,"score":0.4812157452106476},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.41746512055397034},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.39044904708862305},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3305298686027527},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.19755101203918457},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.15703800320625305},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata55660.2022.10020316","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/bigdata55660.2022.10020316","pdf_url":null,"source":{"id":"https://openalex.org/S4363607709","display_name":"2022 IEEE International Conference on Big Data (Big Data)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE International Conference on Big Data (Big Data)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2008503861","https://openalex.org/W2092292480","https://openalex.org/W2119738171","https://openalex.org/W2153972927","https://openalex.org/W2220052790","https://openalex.org/W2539672212","https://openalex.org/W2883188304","https://openalex.org/W2907502740","https://openalex.org/W2949006873","https://openalex.org/W3136130486","https://openalex.org/W3146448298","https://openalex.org/W3168289090","https://openalex.org/W4205500585","https://openalex.org/W6673423357","https://openalex.org/W6753341595","https://openalex.org/W6756770702"],"related_works":["https://openalex.org/W1538652242","https://openalex.org/W2011521129","https://openalex.org/W4379164835","https://openalex.org/W2936171637","https://openalex.org/W1586214342","https://openalex.org/W2260589296","https://openalex.org/W2990494149","https://openalex.org/W3157828377","https://openalex.org/W4290059108","https://openalex.org/W3088424364"],"abstract_inverted_index":{"As":[0],"millions":[1],"of":[2,17,66,68,136,143],"people":[3],"interact":[4],"with":[5,98],"the":[6,54,116,120,134,147,153],"Twitter":[7,46,55],"platform":[8,38],"across":[9],"various":[10],"different":[11,137],"products,":[12],"their":[13,24,157],"actions":[14],"generate":[15],"petabytes":[16,69],"data":[18,42,59,64,71,74,85],"which":[19],"is":[20,33,76],"used":[21],"to":[22,57,146],"make":[23],"experience":[25],"and":[26,48,89,125,139],"usage":[27],"better.":[28],"The":[29],"Data":[30],"Ingestion":[31],"framework":[32],"responsible":[34],"for":[35,39,53,87,107,156],"providing":[36],"a":[37,104],"aggregating":[40],"incoming":[41,70],"from":[43],"services":[44],"backing":[45],"products":[47],"generating":[49],"well":[50],"defined":[51],"datasets":[52],"engineers":[56],"build":[58],"driven":[60],"products.":[61],"At":[62],"Twitter\u2019s":[63],"scale":[65],"tens":[67],"every":[72],"day,":[73],"ingestion":[75,83,92],"offered":[77],"in":[78],"two":[79],"modes.":[80],"Batch":[81],"based":[82,91],"aggregates":[84,93],"optimized":[86],"throughput":[88],"streaming":[90],"events":[94],"as":[95],"they":[96],"arrive,":[97],"low":[99],"latency.":[100,110],"Each":[101],"option":[102,155],"presents":[103],"trade":[105],"off":[106],"cost":[108],"vs":[109],"In":[111],"this":[112],"paper":[113],"we":[114],"describe":[115],"architecture":[117],"behind":[118],"both":[119],"options,":[121],"break":[122],"down":[123],"components":[124],"compare":[126],"them":[127],"against":[128],"important":[129],"metrics.":[130],"We":[131],"talk":[132],"about":[133],"importance":[135],"frameworks":[138],"further":[140],"provide":[141],"details":[142],"tools":[144],"available":[145],"developers,":[148],"who":[149],"can":[150],"then":[151],"choose":[152],"right":[154],"use":[158],"case.":[159]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2025-12-24T23:09:58.560324","created_date":"2025-10-10T00:00:00"}