{"id":"https://openalex.org/W7225408802","doi":"https://doi.org/10.5281/zenodo.23998399","title":"Predictive Batching for Generative AI: Forecasting Future Request Arrivals and Proactively Organizing Computational Workloads","display_name":"Predictive Batching for Generative AI: Forecasting Future Request Arrivals and Proactively Organizing Computational Workloads","publication_year":2026,"publication_date":"2026-11-06","ids":{"openalex":"https://openalex.org/W7225408802","doi":"https://doi.org/10.5281/zenodo.23998399"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23998399","doi":"https://doi.org/10.5281/zenodo.23998399","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23998399","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23998399","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100638045","display_name":"Jincheng Zhang","orcid":"https://orcid.org/0000-0001-7332-6704","observed_orcids":["https://orcid.org/0000-0001-7332-6704","https://orcid.org/0009-0005-1860-0009"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Zhang, Jincheng","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0005-1860-0009","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5100638045"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.3075999915599823,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.3075999915599823,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.2759000062942505,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.2117999941110611,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-inference","display_name":"large language model inference","score":0.7649999856948853},{"id":"https://openalex.org/keywords/gpu-utilization","display_name":"GPU utilization","score":0.6309999823570251},{"id":"https://openalex.org/keywords/batch-scheduling","display_name":"batch scheduling","score":0.8410000205039978},{"id":"https://openalex.org/keywords/continuous-batching","display_name":"continuous batching","score":0.6489999890327454},{"id":"https://openalex.org/keywords/deadline-aware-scheduling","display_name":"deadline-aware scheduling","score":0.8659999966621399},{"id":"https://openalex.org/keywords/prediction-error","display_name":"prediction error","score":0.4230000078678131},{"id":"https://openalex.org/keywords/gpu-memory-constraints","display_name":"GPU memory constraints","score":0.6510000228881836},{"id":"https://openalex.org/keywords/tail-latency","display_name":"tail latency","score":0.5609999895095825},{"id":"https://openalex.org/keywords/energy-consumption","display_name":"energy consumption","score":0.49799999594688416},{"id":"https://openalex.org/keywords/time-to-first-token","display_name":"time-to-first-token","score":0.6980000138282776},{"id":"https://openalex.org/keywords/arrival-time-uncertainty","display_name":"arrival time uncertainty","score":0.6449999809265137},{"id":"https://openalex.org/keywords/service-level-requirements","display_name":"service level requirements","score":0.5450000166893005}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.85589998960495},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.6384999752044678},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5260000228881836},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.5065000057220459},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.49540001153945923},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.4575999975204468},{"id":"https://openalex.org/C172658912","wikidata":"https://www.wikidata.org/wiki/Q661613","display_name":"Batch processing","level":2,"score":0.44339999556541443},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.36559998989105225},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.337799996137619},{"id":"https://openalex.org/C111873713","wikidata":"https://www.wikidata.org/wiki/Q1641413","display_name":"Job scheduler","level":3,"score":0.33629998564720154},{"id":"https://openalex.org/C127964446","wikidata":"https://www.wikidata.org/wiki/Q1092142","display_name":"Computational resource","level":3,"score":0.31679999828338623},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.31119999289512634},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.3037000000476837},{"id":"https://openalex.org/C55416958","wikidata":"https://www.wikidata.org/wiki/Q6206757","display_name":"Job shop scheduling","level":3,"score":0.29490000009536743},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.27810001373291016},{"id":"https://openalex.org/C98025372","wikidata":"https://www.wikidata.org/wiki/Q477538","display_name":"Systems architecture","level":3,"score":0.2741999924182892},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C107568181","wikidata":"https://www.wikidata.org/wiki/Q5319000","display_name":"Dynamic priority scheduling","level":3,"score":0.27059999108314514},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26899999380111694},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2676999866962433},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.2632000148296356},{"id":"https://openalex.org/C176553487","wikidata":"https://www.wikidata.org/wiki/Q7855819","display_name":"Turnaround time","level":2,"score":0.2603999972343445},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.25429999828338623},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23998399","doi":"https://doi.org/10.5281/zenodo.23998399","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23998399","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23998400","doi":"https://doi.org/10.5281/zenodo.23998400","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23998400","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23998399","doi":"https://doi.org/10.5281/zenodo.23998399","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23998399","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7225390238","https://openalex.org/W7225375373","https://openalex.org/W7225378531","https://openalex.org/W7225432460","https://openalex.org/W7225422778","https://openalex.org/W7225376960","https://openalex.org/W7225267314","https://openalex.org/W7225392849","https://openalex.org/W7225376051","https://openalex.org/W7225400169"],"abstract_inverted_index":{"The":[0,192,272,402],"increasing":[1],"deployment":[2],"of":[3,160,242,284,293,420,425,430,489],"generative":[4,465],"artificial":[5],"intelligence":[6],"(AI)":[7],"systems":[8,28,73,467],"has":[9],"created":[10],"substantial":[11],"demand":[12],"for":[13,112,144,248,322,349],"efficient":[14],"GPU":[15,92,186,254,380,395],"utilization,":[16,379],"high-throughput":[17],"inference,":[18],"and":[19,35,90,129,157,176,189,223,225,229,264,287,312,340,372,398,434,484],"low-latency":[20],"request":[21,123,200,279,332,366,382,421],"processing.":[22],"Large":[23],"language":[24],"model":[25],"(LLM)":[26],"inference":[27,59,72],"commonly":[29],"employ":[30],"dynamic":[31],"batching,":[32,34,367,369,371],"continuous":[33,230,304,318,368],"iteration-level":[36],"scheduling":[37,62,105,313,432],"to":[38,131,147,167,237,360,386,437,453,471,482],"combine":[39],"multiple":[40],"requests":[41,53,79,100,138,146,180,326,350,475],"into":[42,181],"computational":[43,133,155,182,208,266,400,487],"batches.":[44],"Although":[45],"these":[46,165],"techniques":[47],"improve":[48],"hardware":[49],"utilization":[50],"by":[51,98],"coordinating":[52],"that":[54,119,184,351,408,476],"have":[55,352,477],"already":[56,478],"entered":[57],"the":[58,99,104,136,151,239,265,282,289,418,423,428,435,486],"system,":[60],"their":[61],"decisions":[63,314],"remain":[64],"substantially":[65],"dependent":[66],"on":[67,417],"currently":[68],"observable":[69],"workloads.":[70,162,491],"Consequently,":[71],"may":[74,308],"encounter":[75],"periods":[76],"in":[77,103,315,463],"which":[78,464],"arrive":[80],"irregularly,":[81],"compatible":[82],"workloads":[83],"are":[84],"distributed":[85],"across":[86],"different":[87],"arrival":[88,124,152,283],"intervals,":[89],"available":[91],"capacity":[93],"cannot":[94],"be":[95,127],"fully":[96],"utilized":[97],"immediately":[101],"present":[102],"queue.":[106],"This":[107],"paper":[108],"introduces":[109],"Predictive":[110],"Batching":[111],"Generative":[113],"AI":[114,466],"(PB-GAI),":[115],"a":[116,294,299,330,459],"conceptual":[117],"framework":[118,273,404],"investigates":[120,458],"whether":[121],"future":[122,161,204,490],"patterns":[125],"can":[126],"predicted":[128,331],"used":[130],"organize":[132,178,485],"batches":[134,473],"before":[135],"corresponding":[137],"materialize.":[139],"Rather":[140],"than":[141],"waiting":[142,383,440],"exclusively":[143],"incoming":[145],"accumulate,":[148],"PB-GAI":[149],"estimates":[150],"times,":[153],"expected":[154,240],"characteristics,":[156],"compatibility":[158,209,424],"relationships":[159],"It":[163,296],"uses":[164],"predictions":[166],"construct":[168],"provisional":[169],"batch":[170,213,217,221,244,251,310,342,378,454],"configurations,":[171],"reserve":[172],"appropriate":[173],"execution":[174,222,252,291],"opportunities,":[175],"dynamically":[177,323],"arriving":[179],"groups":[183],"satisfy":[185],"resource":[187,443],"constraints":[188],"service-level":[190],"requirements.":[191],"proposed":[193,359,403],"architecture":[194],"integrates":[195],"seven":[196],"principal":[197],"functions:":[198],"(1)":[199],"arrival-time":[201,249,447],"forecasting,":[202],"(2)":[203],"workload":[205,285],"characterization,":[206],"(3)":[207],"estimation,":[210],"(4)":[211],"predictive":[212,243,306,362,409],"formation,":[214,455],"(5)":[215],"deadline-aware":[216],"scheduling,":[218],"(6)":[219],"adaptive":[220],"reconstruction,":[224],"(7)":[226],"prediction-error":[227],"feedback":[228],"optimization.":[231],"A":[232],"mathematical":[233],"formulation":[234],"is":[235,333,358,411],"developed":[236],"characterize":[238],"benefits":[241],"formation":[245,343],"while":[246,317],"accounting":[247],"uncertainty,":[250],"efficiency,":[253],"memory":[255],"capacity,":[256],"heterogeneous":[257],"sequence":[258],"lengths,":[259],"waiting-time":[260],"costs,":[261],"deadline":[262,391],"constraints,":[263],"expenditure":[267],"associated":[268],"with":[269,302,364],"incorrect":[270],"predictions.":[271],"further":[274],"distinguishes":[275],"between":[276],"predicting":[277,281,288],"individual":[278],"arrivals,":[280,422],"groups,":[286],"optimal":[290],"time":[292,385],"batch.":[295],"also":[297,481],"establishes":[298],"complementary":[300],"relationship":[301],"conventional":[303,365],"batching:":[305],"batching":[307,319,363,410],"prepare":[309],"configurations":[311],"advance,":[316],"remains":[320],"responsible":[321],"incorporating":[324],"actual":[325,338],"during":[327],"execution.":[328],"Importantly,":[329],"not":[334,345,353,406,469],"treated":[335],"as":[336,449],"an":[337,450],"request,":[339],"speculative":[341,442],"does":[344,405],"require":[346],"generating":[347],"responses":[348],"arrived.":[354],"An":[355],"experimental":[356],"methodology":[357],"compare":[361],"fixed-window":[370],"prediction-assisted":[373],"scheduling.":[374],"Evaluation":[375],"metrics":[376],"include":[377],"throughput,":[381],"time,":[384],"first":[387],"token,":[388],"tail":[389],"latency,":[390],"satisfaction,":[392],"prediction":[393,448],"accuracy,":[394],"energy":[396],"consumption,":[397],"total":[399],"expenditure.":[401],"assume":[407],"universally":[412],"beneficial.":[413],"Its":[414],"effectiveness":[415],"depends":[416],"predictability":[419],"anticipated":[426],"workloads,":[427],"availability":[429],"sufficient":[431],"flexibility,":[433],"ability":[436],"avoid":[438],"unnecessary":[439],"or":[441],"reservation.":[444],"By":[445],"introducing":[446],"additional":[451],"input":[452],"this":[456],"research":[457],"broader":[460],"inference-serving":[461],"paradigm":[462],"attempt":[468],"only":[470],"optimize":[472],"from":[474],"arrived,":[479],"but":[480],"anticipate":[483],"structure":[488]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-11-07T00:00:00"}