{"id":"https://openalex.org/W4401863792","doi":"https://doi.org/10.1145/3637528.3671633","title":"L <scp>umos</scp> : Empowering Multimodal LLMs with Scene Text Recognition","display_name":"L <scp>umos</scp> : Empowering Multimodal LLMs with Scene Text Recognition","publication_year":2024,"publication_date":"2024-08-24","ids":{"openalex":"https://openalex.org/W4401863792","doi":"https://doi.org/10.1145/3637528.3671633"},"language":"en","primary_location":{"id":"doi:10.1145/3637528.3671633","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671633","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671633","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671633","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030041600","display_name":"Ashish Shenoy","orcid":"https://orcid.org/0000-0003-1401-262X"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ashish Shenoy","raw_affiliation_strings":["Meta Reality Labs, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Redmond, WA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101187362","display_name":"Yichao Lu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yichao Lu","raw_affiliation_strings":["Meta Reality Labs, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Redmond, WA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104243951","display_name":"Srihari Jayakumar","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Srihari Jayakumar","raw_affiliation_strings":["Meta Reality Labs, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088724367","display_name":"Debojeet Chatterjee","orcid":"https://orcid.org/0000-0002-2946-3151"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Debojeet Chatterjee","raw_affiliation_strings":["Meta Reality Labs, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068692190","display_name":"Mohsen Moslehpour","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohsen Moslehpour","raw_affiliation_strings":["Meta Reality Labs, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069007377","display_name":"Pierce Chuang","orcid":"https://orcid.org/0000-0001-5850-7048"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pierce Chuang","raw_affiliation_strings":["Meta Reality Labs, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093931906","display_name":"Abhay Harpale","orcid":"https://orcid.org/0000-0002-9503-4881"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Abhay Harpale","raw_affiliation_strings":["Meta Reality Labs, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114213534","display_name":"Vikas Bhardwaj","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vikas Bhardwaj","raw_affiliation_strings":["Meta Reality Labs, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Redmond, WA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108573105","display_name":"Di Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Di Xu","raw_affiliation_strings":["Reality Labs, Meta, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Reality Labs, Meta, Redmond, WA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109768937","display_name":"Shicong Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shicong Zhao","raw_affiliation_strings":["Meta Reality Labs, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Redmond, WA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113887589","display_name":"Longfang Zhao","orcid":"https://orcid.org/0009-0008-8881-9729"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Longfang Zhao","raw_affiliation_strings":["Meta Reality Labs, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Redmond, WA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081037435","display_name":"Ankit Ramchandani","orcid":"https://orcid.org/0000-0002-2843-2505"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ankit Ramchandani","raw_affiliation_strings":["Meta, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091265603","display_name":"Xin Luna Dong","orcid":"https://orcid.org/0009-0001-2049-2458"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xin Luna Dong","raw_affiliation_strings":["Meta Reality Labs, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Redmond, WA, USA","institution_ids":["https://openalex.org/I4210128585"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103700310","display_name":"Anuj Kumar","orcid":"https://orcid.org/0000-0001-7820-9945"},"institutions":[{"id":"https://openalex.org/I4210128585","display_name":"META Health","ror":"https://ror.org/035h67p10","country_code":"US","type":"other","lineage":["https://openalex.org/I4210128585"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anuj Kumar","raw_affiliation_strings":["Meta Reality Labs, Redmond, WA, USA"],"affiliations":[{"raw_affiliation_string":"Meta Reality Labs, Redmond, WA, USA","institution_ids":["https://openalex.org/I4210128585"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":14,"corresponding_author_ids":["https://openalex.org/A5030041600"],"corresponding_institution_ids":["https://openalex.org/I4210128585"],"apc_list":null,"apc_paid":null,"fwci":0.2493,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.51662263,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"5690","last_page":"5700"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10601","display_name":"Handwritten Text Recognition Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10601","display_name":"Handwritten Text Recognition Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7416754961013794},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.7063690423965454},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.6040704846382141},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6039655804634094},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.5254958271980286},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.5041006803512573},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4788508415222168},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.46315494179725647},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4178701341152191},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.40327996015548706},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3734584450721741}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7416754961013794},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.7063690423965454},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.6040704846382141},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6039655804634094},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5254958271980286},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.5041006803512573},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4788508415222168},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.46315494179725647},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4178701341152191},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.40327996015548706},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3734584450721741},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3637528.3671633","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671633","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671633","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3637528.3671633","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671633","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671633","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401863792.pdf"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1922126009","https://openalex.org/W1933349210","https://openalex.org/W2194187530","https://openalex.org/W2593539516","https://openalex.org/W2809273748","https://openalex.org/W2963517393","https://openalex.org/W2963857746","https://openalex.org/W3003484198","https://openalex.org/W3034609471","https://openalex.org/W3106228955","https://openalex.org/W3120043490","https://openalex.org/W3168649818","https://openalex.org/W3172087149","https://openalex.org/W3179897446","https://openalex.org/W3197688480","https://openalex.org/W3200114912","https://openalex.org/W4226237627","https://openalex.org/W4296068796","https://openalex.org/W4312747027"],"related_works":["https://openalex.org/W2384605597","https://openalex.org/W2387743295","https://openalex.org/W2115758952","https://openalex.org/W3082787378","https://openalex.org/W2136007095","https://openalex.org/W2366230879","https://openalex.org/W3208425359","https://openalex.org/W2349927912","https://openalex.org/W3128807919","https://openalex.org/W3176411177"],"abstract_inverted_index":{"We":[0,88],"introduce":[1],"Lumos,":[2,51],"the":[3,14,33,75],"first":[4,29],"end-to-end":[5],"multimodal":[6],"question-answering":[7],"system":[8,76],"with":[9],"text":[10,27],"understanding":[11],"capabilities.":[12],"At":[13],"core":[15],"of":[16,35],"Lumos":[17],"is":[18,37],"a":[19,43,91],"Scene":[20],"Text":[21],"Recognition":[22],"(STR)":[23],"component":[24],"that":[25],"extracts":[26],"from":[28],"person":[30],"point-of-view":[31],"images,":[32],"output":[34],"which":[36],"used":[38],"to":[39,42,57,84],"augment":[40],"input":[41],"Multimodal":[44],"Large":[45],"Language":[46],"Model":[47],"(MM-LLM).":[48],"While":[49],"building":[50],"we":[52,68],"encountered":[53],"numerous":[54],"challenges":[55],"related":[56],"STR":[58],"quality,":[59],"overall":[60],"latency,":[61],"and":[62,73,80,100],"model":[63],"inference.":[64],"In":[65],"this":[66],"paper,":[67],"delve":[69],"into":[70],"those":[71],"challenges,":[72],"discuss":[74],"architecture,":[77],"design":[78],"choices,":[79],"modeling":[81],"techniques":[82],"employed":[83],"overcome":[85],"these":[86],"obstacles.":[87],"also":[89],"provide":[90],"comprehensive":[92],"evaluation":[93],"for":[94],"each":[95],"component,":[96],"showcasing":[97],"high":[98],"quality":[99],"efficiency.":[101]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}