{"id":"https://openalex.org/W4387596061","doi":"https://doi.org/10.48550/arxiv.2310.07160","title":"LLark: A Multimodal Instruction-Following Language Model for Music","display_name":"LLark: A Multimodal Instruction-Following Language Model for Music","publication_year":2023,"publication_date":"2023-10-11","ids":{"openalex":"https://openalex.org/W4387596061","doi":"https://doi.org/10.48550/arxiv.2310.07160"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.07160","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.07160","pdf_url":"https://arxiv.org/pdf/2310.07160","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.07160","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5110941464","display_name":"Josh Gardner","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gardner, Josh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108294493","display_name":"Simon Durand","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Durand, Simon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063832676","display_name":"Daniel Stoller","orcid":"https://orcid.org/0000-0002-8615-4144","observed_orcids":["https://orcid.org/0000-0002-8615-4144"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stoller, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5019790225","display_name":"Rachel Bittner","orcid":"https://orcid.org/0000-0001-7757-2232","observed_orcids":["https://orcid.org/0000-0001-7757-2232"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bittner, Rachel M.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":47,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9247999787330627,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9247999787330627,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.06679999828338623,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11349","display_name":"Music Technology and Sound Studies","score":0.004399999976158142,"subfield":{"id":"https://openalex.org/subfields/1210","display_name":"Music"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/musical-understanding","display_name":"musical understanding","score":0.796999990940094},{"id":"https://openalex.org/keywords/instruction-tuned-language-models","display_name":"instruction-tuned language models","score":0.6990000009536743},{"id":"https://openalex.org/keywords/multimodal-models","display_name":"multimodal models","score":0.9039999842643738},{"id":"https://openalex.org/keywords/music-captioning","display_name":"music captioning","score":0.8230000138282776}],"concepts":[{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.864844799041748},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.437224805355072},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.15231487154960632},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.052054792642593384}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.07160","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.07160","pdf_url":"https://arxiv.org/pdf/2310.07160","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.07160","doi":"https://doi.org/10.48550/arxiv.2310.07160","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.07160","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.07160","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.07160","pdf_url":"https://arxiv.org/pdf/2310.07160","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387596061.pdf","grobid_xml":"https://content.openalex.org/works/W4387596061.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2405656250","https://openalex.org/W2524365899","https://openalex.org/W2885570506","https://openalex.org/W2922872820","https://openalex.org/W3021164770","https://openalex.org/W3184977056","https://openalex.org/W3198002980","https://openalex.org/W4304699761","https://openalex.org/W4318351475","https://openalex.org/W4366330503","https://openalex.org/W4366850747","https://openalex.org/W4367628410","https://openalex.org/W4377086529","https://openalex.org/W4377130946","https://openalex.org/W4377372369","https://openalex.org/W4379251869","https://openalex.org/W4384918448","https://openalex.org/W4385572634","https://openalex.org/W4385807453","https://openalex.org/W4386555501","https://openalex.org/W4387210470","https://openalex.org/W4392909390","https://openalex.org/W4415797148","https://openalex.org/W7133189643","https://openalex.org/W7133239970","https://openalex.org/W7133243546"],"related_works":["https://openalex.org/W4416036374","https://openalex.org/W7214231132","https://openalex.org/W4417299246","https://openalex.org/W7101888949","https://openalex.org/W4416057713","https://openalex.org/W7165115836","https://openalex.org/W4401978739","https://openalex.org/W7125135325","https://openalex.org/W4401044042","https://openalex.org/W4416238822"],"abstract_inverted_index":{"Music":[0],"has":[1],"a":[2,61,67,73,80,111],"unique":[3,22],"and":[4,14,20,57,107,121,132,134,150,156],"complex":[5],"structure":[6],"which":[7,47],"is":[8,125,160],"challenging":[9],"for":[10,37,44,70,77],"both":[11],"expert":[12],"humans":[13,109],"existing":[15,102],"AI":[16],"systems":[17],"to":[18,25,60],"understand,":[19],"presents":[21],"challenges":[23],"relative":[24],"other":[26],"forms":[27],"of":[28,52,89,114,145],"audio.":[29],"We":[30,40,65],"present":[31],"LLark,":[32,71],"an":[33],"instruction-tuned":[34],"multimodal":[35,68],"model":[36,76],"\\emph{music}":[38],"understanding.":[39],"detail":[41],"our":[42,137,157],"process":[43],"dataset":[45],"creation,":[46],"involves":[48],"augmenting":[49],"the":[50,143],"annotations":[51],"diverse":[53],"open-source":[54,129],"music":[55,78,105,130],"datasets":[56],"converting":[58],"them":[59],"unified":[62],"instruction-tuning":[63],"format.":[64],"propose":[66],"architecture":[69],"integrating":[72],"pretrained":[74,81],"generative":[75],"with":[79,116,142],"language":[82],"model.":[83],"In":[84],"evaluations":[85],"on":[86],"three":[87],"types":[88],"tasks":[90],"(music":[91],"understanding,":[92,106],"captioning,":[93],"reasoning),":[94],"we":[95,135],"show":[96,110],"that":[97,108],"LLark":[98,124],"matches":[99],"or":[100],"outperforms":[101],"baselines":[103],"in":[104,119],"high":[112],"degree":[113],"agreement":[115],"its":[117],"responses":[118],"captioning":[120],"reasoning":[122],"tasks.":[123],"trained":[126],"entirely":[127],"from":[128],"data":[131],"models,":[133],"make":[136],"training":[138],"code":[139,159],"available":[140,161],"along":[141],"release":[144],"this":[146],"paper.":[147],"Additional":[148],"results":[149],"audio":[151],"examples":[152],"are":[153],"at":[154,162],"https://bit.ly/llark,":[155],"source":[158],"https://github.com/spotify-research/llark":[163],".":[164]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":17},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-10-13T00:00:00"}