{"id":"https://openalex.org/W3196689789","doi":"https://doi.org/10.1145/3474085.3475557","title":"Sensor-Augmented Egocentric-Video Captioning with Dynamic Modal Attention","display_name":"Sensor-Augmented Egocentric-Video Captioning with Dynamic Modal Attention","publication_year":2021,"publication_date":"2021-10-17","ids":{"openalex":"https://openalex.org/W3196689789","doi":"https://doi.org/10.1145/3474085.3475557","mag":"3196689789"},"language":"en","primary_location":{"id":"doi:10.1145/3474085.3475557","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3474085.3475557","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3474085.3475557","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3474085.3475557","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007900897","display_name":"Katsuyuki Nakamura","orcid":"https://orcid.org/0000-0002-8074-2279"},"institutions":[{"id":"https://openalex.org/I65143321","display_name":"Hitachi (Japan)","ror":"https://ror.org/02exqgm79","country_code":"JP","type":"company","lineage":["https://openalex.org/I65143321"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Katsuyuki Nakamura","raw_affiliation_strings":["Hitachi, Ltd., Kokubunji-shi, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"Hitachi, Ltd., Kokubunji-shi, Tokyo, Japan","institution_ids":["https://openalex.org/I65143321"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112657316","display_name":"Hiroki Ohashi","orcid":null},"institutions":[{"id":"https://openalex.org/I65143321","display_name":"Hitachi (Japan)","ror":"https://ror.org/02exqgm79","country_code":"JP","type":"company","lineage":["https://openalex.org/I65143321"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Hiroki Ohashi","raw_affiliation_strings":["Hitachi, Ltd., Kokubunji-shi, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"Hitachi, Ltd., Kokubunji-shi, Tokyo, Japan","institution_ids":["https://openalex.org/I65143321"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102797548","display_name":"Mitsuhiro Okada","orcid":"https://orcid.org/0000-0001-8869-0760"},"institutions":[{"id":"https://openalex.org/I65143321","display_name":"Hitachi (Japan)","ror":"https://ror.org/02exqgm79","country_code":"JP","type":"company","lineage":["https://openalex.org/I65143321"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Mitsuhiro Okada","raw_affiliation_strings":["Hitachi, Ltd., Kokubunji-shi, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"Hitachi, Ltd., Kokubunji-shi, Tokyo, Japan","institution_ids":["https://openalex.org/I65143321"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5007900897"],"corresponding_institution_ids":["https://openalex.org/I65143321"],"apc_list":null,"apc_paid":null,"fwci":0.5827,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.68733141,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"4220","last_page":"4229"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.9148451089859009},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8337005376815796},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.671161413192749},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6417230367660522},{"id":"https://openalex.org/keywords/wearable-computer","display_name":"Wearable computer","score":0.6129865646362305},{"id":"https://openalex.org/keywords/sensor-fusion","display_name":"Sensor fusion","score":0.5954360365867615},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5721632838249207},{"id":"https://openalex.org/keywords/fuse","display_name":"Fuse (electrical)","score":0.4600399136543274},{"id":"https://openalex.org/keywords/motion","display_name":"Motion (physics)","score":0.45630162954330444},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.12315940856933594}],"concepts":[{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.9148451089859009},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8337005376815796},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.671161413192749},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6417230367660522},{"id":"https://openalex.org/C150594956","wikidata":"https://www.wikidata.org/wiki/Q1334829","display_name":"Wearable computer","level":2,"score":0.6129865646362305},{"id":"https://openalex.org/C33954974","wikidata":"https://www.wikidata.org/wiki/Q486494","display_name":"Sensor fusion","level":2,"score":0.5954360365867615},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5721632838249207},{"id":"https://openalex.org/C141353440","wikidata":"https://www.wikidata.org/wiki/Q182221","display_name":"Fuse (electrical)","level":2,"score":0.4600399136543274},{"id":"https://openalex.org/C104114177","wikidata":"https://www.wikidata.org/wiki/Q79782","display_name":"Motion (physics)","level":2,"score":0.45630162954330444},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.12315940856933594},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3474085.3475557","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3474085.3475557","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3474085.3475557","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2109.02955","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2109.02955","pdf_url":"https://arxiv.org/pdf/2109.02955","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3474085.3475557","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3474085.3475557","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3474085.3475557","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Multimedia","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2983817602","display_name":null,"funder_award_id":"EEC-0540865","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3323168326","display_name":null,"funder_award_id":"0540865","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5227708377","display_name":null,"funder_award_id":"EEEC-0540865","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3196689789.pdf","grobid_xml":"https://content.openalex.org/works/W3196689789.grobid-xml"},"referenced_works_count":117,"referenced_works":["https://openalex.org/W648786980","https://openalex.org/W1514535095","https://openalex.org/W1573040851","https://openalex.org/W1586939924","https://openalex.org/W1811254738","https://openalex.org/W1821462560","https://openalex.org/W1828348983","https://openalex.org/W1889081078","https://openalex.org/W1895577753","https://openalex.org/W1895989618","https://openalex.org/W1940585053","https://openalex.org/W1947050545","https://openalex.org/W1947481528","https://openalex.org/W1948812921","https://openalex.org/W1967686239","https://openalex.org/W1969616664","https://openalex.org/W2016053056","https://openalex.org/W2019273110","https://openalex.org/W2031688197","https://openalex.org/W2038219201","https://openalex.org/W2070999216","https://openalex.org/W2105046342","https://openalex.org/W2106229755","https://openalex.org/W2109586012","https://openalex.org/W2110933980","https://openalex.org/W2135450947","https://openalex.org/W2136036867","https://openalex.org/W2136668269","https://openalex.org/W2139501017","https://openalex.org/W2142900973","https://openalex.org/W2147806277","https://openalex.org/W2149172860","https://openalex.org/W2149276562","https://openalex.org/W2162762857","https://openalex.org/W2163915297","https://openalex.org/W2165605600","https://openalex.org/W2173180041","https://openalex.org/W2197223256","https://openalex.org/W2198667788","https://openalex.org/W2230466109","https://openalex.org/W2295523511","https://openalex.org/W2337676748","https://openalex.org/W2341342588","https://openalex.org/W2387799167","https://openalex.org/W2461911683","https://openalex.org/W2473032611","https://openalex.org/W2512308465","https://openalex.org/W2523993696","https://openalex.org/W2547875792","https://openalex.org/W2557264465","https://openalex.org/W2560662850","https://openalex.org/W2575842049","https://openalex.org/W2584992898","https://openalex.org/W2593390416","https://openalex.org/W2610163825","https://openalex.org/W2626778328","https://openalex.org/W2737602873","https://openalex.org/W2742841012","https://openalex.org/W2760103357","https://openalex.org/W2766375149","https://openalex.org/W2790888757","https://openalex.org/W2793340237","https://openalex.org/W2796136333","https://openalex.org/W2887053183","https://openalex.org/W2896457183","https://openalex.org/W2914587137","https://openalex.org/W2931316642","https://openalex.org/W2948772644","https://openalex.org/W2949447259","https://openalex.org/W2950019618","https://openalex.org/W2950178297","https://openalex.org/W2950304420","https://openalex.org/W2950307714","https://openalex.org/W2951390634","https://openalex.org/W2952296662","https://openalex.org/W2962706528","https://openalex.org/W2963221612","https://openalex.org/W2963351113","https://openalex.org/W2963381281","https://openalex.org/W2963403868","https://openalex.org/W2963410018","https://openalex.org/W2963916161","https://openalex.org/W2963971014","https://openalex.org/W2964222622","https://openalex.org/W2964241990","https://openalex.org/W2964242760","https://openalex.org/W2964253311","https://openalex.org/W2964532449","https://openalex.org/W2968101724","https://openalex.org/W2970754407","https://openalex.org/W2981851019","https://openalex.org/W2984138079","https://openalex.org/W2985144848","https://openalex.org/W2991451943","https://openalex.org/W2997004687","https://openalex.org/W3005037856","https://openalex.org/W3009150298","https://openalex.org/W3025136821","https://openalex.org/W3025796084","https://openalex.org/W3034593503","https://openalex.org/W3034667697","https://openalex.org/W3034815696","https://openalex.org/W3034891989","https://openalex.org/W3035237998","https://openalex.org/W3035365026","https://openalex.org/W3035367723","https://openalex.org/W3035392611","https://openalex.org/W3045031188","https://openalex.org/W3094751268","https://openalex.org/W3098401379","https://openalex.org/W3098682680","https://openalex.org/W3105479157","https://openalex.org/W3128093669","https://openalex.org/W4239511627","https://openalex.org/W4287777632","https://openalex.org/W4299341000","https://openalex.org/W4385245566"],"related_works":["https://openalex.org/W4210416330","https://openalex.org/W2775506363","https://openalex.org/W3088136942","https://openalex.org/W4290852288","https://openalex.org/W2949362007","https://openalex.org/W4388893791","https://openalex.org/W4283207562","https://openalex.org/W2963177403","https://openalex.org/W2330246314","https://openalex.org/W2949522393"],"abstract_inverted_index":{"Automatically":[0],"describing":[1],"video,":[2],"or":[3,51],"video":[4,47,58,141],"captioning,":[5,23],"has":[6,82],"been":[7],"widely":[8],"studied":[9],"in":[10,63,118,137],"the":[11,37,73,90,97,115,134,140,144,153,160,167,175,189,200,204,207],"multimedia":[12],"field.":[13],"This":[14],"paper":[15],"proposes":[16],"a":[17,24,34,77,100,129],"new":[18],"task":[19,40],"of":[20,46,68,76,93,99,146,206],"sensor-augmented":[21],"egocentric-video":[22,190],"newly":[25,38],"constructed":[26],"dataset":[27,178],"for":[28,36,88,131],"it":[29],"called":[30],"MMAC":[31,176],"Captions,":[32],"and":[33,48,124,179,194],"method":[35,130,170,198],"proposed":[39,168,197,208],"that":[41,150,155,181,195],"effectively":[42,132],"utilizes":[43],"multi-modal":[44],"data":[45,109,136,142,184,191],"motion":[49,121],"sensors,":[50],"inertial":[52],"measurement":[53],"units":[54],"(IMUs).":[55],"While":[56],"conventional":[57],"captioning":[59],"tasks":[60],"have":[61],"difficulty":[62],"dealing":[64],"with":[65,139,171],"detailed":[66],"descriptions":[67,92],"human":[69,94],"activities":[70,95],"due":[71],"to":[72,85,113,188],"limited":[74],"view":[75],"fixed":[78],"camera,":[79],"egocentric":[80,119],"vision":[81],"greater":[83],"potential":[84],"be":[86],"used":[87],"generating":[89],"finer-grained":[91],"on":[96,143,174],"basis":[98,145],"much":[101],"closer":[102],"view.":[103],"In":[104],"addition,":[105],"we":[106],"utilize":[107],"wearable-sensor":[108],"as":[110,185],"auxiliary":[111],"information":[112,162,187],"mitigate":[114],"inherent":[116],"problems":[117],"vision:":[120],"blur,":[122],"self-occlusion,":[123],"out-of-camera-range":[125],"activities.":[126],"We":[127,165],"propose":[128],"utilizing":[133],"sensor":[135,183],"combination":[138],"an":[147],"attention":[148],"mechanism":[149],"dynamically":[151],"determines":[152],"modality":[154],"requires":[156],"more":[157],"attention,":[158],"taking":[159],"contextual":[161],"into":[163],"account.":[164],"compared":[166],"sensor-fusion":[169],"strong":[172,201],"baselines":[173],"Captions":[177],"found":[180],"using":[182],"supplementary":[186],"was":[192],"beneficial,":[193],"our":[196],"outperformed":[199],"baselines,":[202],"demonstrating":[203],"effectiveness":[205],"method.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":2}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2021-09-13T00:00:00"}