{"id":"https://openalex.org/W4403713288","doi":"https://doi.org/10.1145/3689091.3690086","title":"Shotluck Holmes: A Family of Efficient Small-Scale Large Language Vision Models For Video Captioning and Summarization","display_name":"Shotluck Holmes: A Family of Efficient Small-Scale Large Language Vision Models For Video Captioning and Summarization","publication_year":2024,"publication_date":"2024-10-23","ids":{"openalex":"https://openalex.org/W4403713288","doi":"https://doi.org/10.1145/3689091.3690086"},"language":"en","primary_location":{"id":"doi:10.1145/3689091.3690086","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3689091.3690086","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd International Workshop on Deep Multimodal Generation and Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101295107","display_name":"Richard Luo","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Richard Luo","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, Georgia, USA"],"raw_orcid":"https://orcid.org/0009-0003-3175-6118","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, Georgia, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090330767","display_name":"Austin Peng","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Austin Peng","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, Georgia, USA"],"raw_orcid":"https://orcid.org/0009-0009-8806-4714","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, Georgia, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099008074","display_name":"Adithya Vasudev","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adithya Vasudev","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, Georgia, USA"],"raw_orcid":"https://orcid.org/0009-0002-6146-1241","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, Georgia, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":null,"display_name":"Rishabh Jain","orcid":"https://orcid.org/0009-0007-0481-1280"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rishabh Jain","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, Georgia, USA"],"raw_orcid":"https://orcid.org/0009-0007-0481-1280","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, Georgia, USA","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I130701444"],"apc_list":null,"apc_paid":null,"fwci":0.4285,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.63025722,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"7","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9860000014305115,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.9568691849708557},{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.9439229965209961},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7552511692047119},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5630351901054382},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4667067527770996},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4140962064266205},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3375365734100342},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.15924295783042908},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.08762577176094055},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.0813082754611969}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.9568691849708557},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.9439229965209961},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7552511692047119},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5630351901054382},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4667067527770996},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4140962064266205},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3375365734100342},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.15924295783042908},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.08762577176094055},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.0813082754611969}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3689091.3690086","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3689091.3690086","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd International Workshop on Deep Multimodal Generation and Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6100000143051147}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W2101105183","https://openalex.org/W2560730294","https://openalex.org/W4224308101","https://openalex.org/W4226289673","https://openalex.org/W4296605665","https://openalex.org/W4390873312","https://openalex.org/W6810081322"],"related_works":["https://openalex.org/W4210416330","https://openalex.org/W2775506363","https://openalex.org/W3088136942","https://openalex.org/W4290852288","https://openalex.org/W2949362007","https://openalex.org/W4283207562","https://openalex.org/W2963177403","https://openalex.org/W2330246314","https://openalex.org/W2949522393","https://openalex.org/W4289422896"],"abstract_inverted_index":{"Video":[0],"is":[1,36],"an":[2],"increasingly":[3],"prominent":[4],"and":[5,53,127,136,181,187],"information-dense":[6],"medium,":[7],"yet":[8],"it":[9],"poses":[10],"substantial":[11],"challenges":[12],"for":[13],"language":[14,119],"models.":[15,191],"A":[16],"typical":[17],"video":[18,64,125,179],"consists":[19],"of":[20,23,48,61,72,116,144,162],"a":[21,31,39,42,89,114,153,160],"sequence":[22,161],"shorter":[24],"segments,":[25],"or":[26],"shots,":[27],"that":[28,78,167],"collectively":[29],"form":[30],"coherent":[32],"narrative.":[33],"Each":[34],"shot":[35,74,86],"analogous":[37],"to":[38,87,123,151,155,158],"word":[40],"in":[41,96],"sentence":[43],"where":[44],"multiple":[45],"data":[46,137],"streams":[47],"information":[49,71],"(such":[50],"as":[51],"visual":[52],"auditory":[54],"data)":[55],"must":[56],"be":[57],"processed":[58],"simultaneously.":[59],"Comprehension":[60],"the":[62,69,79,82,97,142,177],"entire":[63],"requires":[65,77],"not":[66],"only":[67],"understanding":[68],"visual-audio":[70],"each":[73,85],"but":[75],"also":[76],"model":[80],"links":[81],"ideas":[83],"between":[84],"generate":[88],"larger,":[90],"all-encompassing":[91],"story.":[92],"Despite":[93],"significant":[94],"progress":[95],"field,":[98],"current":[99],"works":[100],"often":[101],"overlook":[102],"videos'":[103],"more":[104,188],"granular":[105],"shot-by-shot":[106],"semantic":[107],"information.":[108],"In":[109],"this":[110],"project,":[111],"we":[112,140,165],"propose":[113],"family":[115],"efficient":[117,190],"large":[118],"vision":[120],"models":[121],"(LLVMs)":[122],"boost":[124],"summarization":[126],"captioning":[128,180],"called":[129],"Shotluck":[130,168],"Holmes.":[131],"By":[132],"leveraging":[133],"better":[134,171],"pretraining":[135],"collection":[138],"strategies,":[139],"extend":[141],"abilities":[143],"existing":[145],"small":[146],"LLVMs":[147],"from":[148],"being":[149,156],"able":[150,157],"understand":[152,159],"picture":[154],"frames.":[163],"Specifically,":[164],"show":[166],"Holmes":[169],"achieves":[170],"performance":[172],"than":[173],"state-of-the-art":[174],"results":[175],"on":[176],"Shot2Story":[178],"summary":[182],"task":[183],"with":[184],"significantly":[185],"smaller":[186],"computationally":[189]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-10-10T00:00:00"}