{"id":"https://openalex.org/W7220803732","doi":null,"title":"Training a massively multimodal transformer on YouTube data: pre-training and parameter efficient fine-tuning on HPC infrastructure","display_name":"Training a massively multimodal transformer on YouTube data: pre-training and parameter efficient fine-tuning on HPC infrastructure","publication_year":2023,"publication_date":"2023-05-01","ids":{"openalex":"https://openalex.org/W7220803732"},"language":"en","primary_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/120172","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/120172","pdf_url":"https://www.ideals.illinois.edu/items/127393/bitstreams/426905/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ideals.illinois.edu/items/127393/bitstreams/426905/data.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5157712154","display_name":"Kastan Vrabel Day","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Day, Kastan Vrabel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5157712154"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7310000061988831,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7310000061988831,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.23180000483989716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.01600000075995922,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7390000224113464},{"id":"https://openalex.org/keywords/pretraining","display_name":"pre-training","score":0.5370000004768372},{"id":"https://openalex.org/keywords/parameter-efficient-fine-tuning","display_name":"parameter-efficient fine-tuning","score":0.9869999885559082},{"id":"https://openalex.org/keywords/multimodal-transformer","display_name":"multimodal transformer","score":0.7839999794960022},{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.8820000290870667},{"id":"https://openalex.org/keywords/reward-modeling","display_name":"reward modeling","score":0.6309999823570251},{"id":"https://openalex.org/keywords/hallucinations","display_name":"hallucinations","score":0.6859999895095825},{"id":"https://openalex.org/keywords/video-question-answering","display_name":"video question answering","score":0.7519999742507935},{"id":"https://openalex.org/keywords/information-leakage","display_name":"information leakage","score":0.9229999780654907},{"id":"https://openalex.org/keywords/time-series-forecasting","display_name":"time series forecasting","score":0.9100000262260437},{"id":"https://openalex.org/keywords/embedding-space-alignment","display_name":"embedding alignment","score":0.7459999918937683},{"id":"https://openalex.org/keywords/hpc-infrastructure","display_name":"HPC infrastructure","score":0.5759999752044678},{"id":"https://openalex.org/keywords/multi-cloud-systems","display_name":"multi-cloud systems","score":0.6100000143051147},{"id":"https://openalex.org/keywords/open-source-llms","display_name":"open-source LLMs","score":0.8790000081062317}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6794000267982483},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5432000160217285},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4894999861717224},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.321399986743927},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2824000120162964},{"id":"https://openalex.org/C190475519","wikidata":"https://www.wikidata.org/wiki/Q544384","display_name":"Massively parallel","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.26820001006126404},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.2646999955177307},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2612000107765198},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.24899999797344208}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:www.ideals.illinois.edu:2142/120172","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/120172","pdf_url":"https://www.ideals.illinois.edu/items/127393/bitstreams/426905/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/120172","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/2142/120172","pdf_url":"https://www.ideals.illinois.edu/items/127393/bitstreams/426905/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220803732.pdf","grobid_xml":"https://content.openalex.org/works/W7220803732.grobid-xml"},"referenced_works_count":51,"referenced_works":["https://openalex.org/W2742343242","https://openalex.org/W2805003733","https://openalex.org/W2963541336","https://openalex.org/W3030163527","https://openalex.org/W3030520226","https://openalex.org/W3034636873","https://openalex.org/W3092172514","https://openalex.org/W3094550259","https://openalex.org/W3112689365","https://openalex.org/W3118781290","https://openalex.org/W3129831491","https://openalex.org/W3134144764","https://openalex.org/W3139080614","https://openalex.org/W3152956381","https://openalex.org/W3159481202","https://openalex.org/W3168867926","https://openalex.org/W3174770825","https://openalex.org/W3177318507","https://openalex.org/W3177828909","https://openalex.org/W3190965961","https://openalex.org/W3201869122","https://openalex.org/W3205270560","https://openalex.org/W3205803342","https://openalex.org/W4224308101","https://openalex.org/W4225323055","https://openalex.org/W4226271314","https://openalex.org/W4285429195","https://openalex.org/W4298185919","https://openalex.org/W4302011281","https://openalex.org/W4304014690","https://openalex.org/W4306820534","https://openalex.org/W4307079201","https://openalex.org/W4307868887","https://openalex.org/W4309867694","https://openalex.org/W4310416691","https://openalex.org/W4311991106","https://openalex.org/W4312960807","https://openalex.org/W4318718936","https://openalex.org/W4318908031","https://openalex.org/W4321636575","https://openalex.org/W4323557204","https://openalex.org/W4327810158","https://openalex.org/W4330338093","https://openalex.org/W4360836968","https://openalex.org/W4361229539","https://openalex.org/W4361866125","https://openalex.org/W4366736278","https://openalex.org/W4386066385","https://openalex.org/W4396895148","https://openalex.org/W6931203431","https://openalex.org/W7133224126"],"related_works":["https://openalex.org/W7220756504","https://openalex.org/W7220810479","https://openalex.org/W7196946098","https://openalex.org/W6930884629","https://openalex.org/W6912712995","https://openalex.org/W7055789494","https://openalex.org/W7166853303","https://openalex.org/W4385565577","https://openalex.org/W3211458429","https://openalex.org/W3014020888"],"abstract_inverted_index":{"In":[0],"machine":[1],"learning,":[2],"the":[3,20,26,30,49,63,114,152,173,180,183,193,234,252,255,268,279,282],"widespread":[4],"adoption":[5],"of":[6,32,51,65,116,136,154,175,212,225,236,257,271,286],"pre-trained":[7],"large":[8,34],"language":[9,35],"models":[10,36],"(LLMs)":[11],"across":[12],"many":[13],"domains":[14],"has":[15],"disrupted":[16],"conventional":[17],"wisdom":[18],"on":[19,247],"right":[21],"model":[22,205],"to":[23,90,111,143,150,168,266],"use":[24,64],"for":[25,54,93,178,215,241,262,281],"job.This":[27],"work":[28,78,126],"investigates":[29],"advantages":[31],"pre-training":[33,74],"from":[37],"scratch":[38],"over":[39],"fine-tuning":[40,84],"existing":[41],"ones":[42],"in":[43,67,102,189,197,233,288],"specific":[44],"scenarios,":[45],"such":[46,70],"as":[47,71,220],"exploring":[48],"importance":[50,224],"custom":[52],"tokenizers":[53],"domain-specific":[55],"applications,":[56],"addressing":[57],"information":[58],"leakage":[59],"concerns,":[60],"and":[61,95,98,165,192,229,243,249,260,264,274],"examining":[62],"LLMs":[66,259],"non-traditional":[68],"applications":[69],"time-series":[72],"forecasting.When":[73],"is":[75,109,141,231],"unnecessary,":[76],"this":[77],"argues":[79,127],"that":[80,128,171,277],"select":[81],"parameter":[82],"efficient":[83],"(PEFT)":[85],"methods":[86],"are":[87],"strictly":[88],"superior":[89],"traditional-fine":[91],"tuning":[92,139],"data":[94,245],"computational":[96],"efficiency":[97],"should":[99,148],"be":[100],"preferred":[101],"nearly":[103],"all":[104],"cases.Furthermore,":[105],"after":[106],"PEFT,":[107,272],"it":[108],"ideal":[110],"further":[112],"sculpt":[113],"outputs":[115],"one's":[117],"LLM":[118],"with":[119,122,158],"Reinforcement":[120],"Learning":[121],"Human":[123],"Feedback":[124],"(RLHF).This":[125],"reinforcement":[129],"learning":[130,187],"(RL),":[131],"rather":[132],"than":[133],"any":[134],"form":[135],"supervised":[137],"fine":[138],"(SFT),":[140],"preferable":[142],"achieve":[144],"truthfulness":[145],"without":[146],"hallucination.Practitioners":[147],"seek":[149],"leverage":[151],"benefits":[153,174],"RLHF":[155],"via":[156],"RL":[157],"AI":[159,239,289],"feedback":[160,170],"(RLAIF),":[161],"an":[162],"effective,":[163],"fast,":[164],"economical":[166],"alternative":[167],"human":[169],"retains":[172],"reward":[176],"modeling":[177],"factuality.Additionally,":[179],"paper":[181,253],"discusses":[182],"three":[184],"most":[185],"successful":[186],"objectives":[188],"multimodal":[190],"transformers":[191,276],"challenges":[194],"they":[195],"face":[196],"aligning":[198],"distinct":[199],"embedding":[200],"spaces.I":[201],"present":[202],"my":[203],"own":[204],"Video":[206],"Pre-trained":[207],"Transformer:":[208],"A":[209],"Multimodal":[210],"Mixture":[211],"Pre-Trained":[213],"Experts":[214],"video":[216],"question":[217],"answering":[218],"tasks":[219],"benchmarked":[221],"against":[222],"VQAv2.The":[223],"modern":[226],"MLfirst":[227],"databases":[228],"filesystems":[230],"explored":[232],"context":[235],"multimodal,":[237],"multi-model":[238],"systems":[240],"fast":[242],"flexible":[244],"throughput":[246],"HPC":[248],"multi-cloud":[250],"systems.Together,":[251],"represents":[254],"state":[256],"open-source":[258],"opportunities":[261],"researchers":[263],"practitioners":[265],"combine":[267],"attractive":[269],"properties":[270],"RLAIF,":[273],"multi-modal":[275],"paves":[278],"way":[280],"next":[283],"few":[284],"years":[285],"growth":[287],"capabilities.":[290]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-10-07T00:00:00"}