{"id":"https://openalex.org/W6910789561","doi":"https://doi.org/10.48550/arxiv.2506.17781","title":"Beyond instruction-conditioning, MoTE: Mixture of Task Experts for Multi-task Embedding Models","display_name":"Beyond instruction-conditioning, MoTE: Mixture of Task Experts for Multi-task Embedding Models","publication_year":2025,"publication_date":"2025-06-21","ids":{"openalex":"https://openalex.org/W6910789561","doi":"https://doi.org/10.48550/arxiv.2506.17781"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2506.17781","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.17781","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2506.17781","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Romero, Miguel","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Romero, Miguel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ding, Shuoyang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Shuoyang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Barret, Corey D.","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barret, Corey D.","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Dinu, Georgiana","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dinu, Georgiana","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Karypis, George","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karypis, George","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5738000273704529,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5738000273704529,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.10689999908208847,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.08860000222921371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.7572000026702881},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6967999935150146},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6104999780654907},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5953999757766724},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5259000062942505},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.462799996137619},{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"Multi-task learning","score":0.45840001106262207}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7620000243186951},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.7572000026702881},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6967999935150146},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.638700008392334},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6104999780654907},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5953999757766724},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.535099983215332},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5259000062942505},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.462799996137619},{"id":"https://openalex.org/C28006648","wikidata":"https://www.wikidata.org/wiki/Q6934509","display_name":"Multi-task learning","level":3,"score":0.45840001106262207},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.42800000309944153},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.391400009393692},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.38920000195503235},{"id":"https://openalex.org/C2777472644","wikidata":"https://www.wikidata.org/wiki/Q16968992","display_name":"Approximate inference","level":3,"score":0.3718000054359436},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.30160000920295715},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.2935999929904938},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.26489999890327454},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.25369998812675476}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2506.17781","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.17781","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2506.17781","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.17781","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"score":0.6590942740440369,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Dense":[0],"embeddings":[1],"are":[2,112],"fundamental":[3,35],"to":[4,31,73,78],"modern":[5],"machine":[6],"learning":[7],"systems,":[8],"powering":[9],"Retrieval-Augmented":[10],"Generation":[11],"(RAG),":[12],"information":[13],"retrieval,":[14],"and":[15,53,98],"representation":[16],"learning.":[17],"While":[18],"instruction-conditioning":[19],"has":[20],"become":[21],"the":[22,40,55,75],"dominant":[23],"approach":[24],"for":[25],"embedding":[26],"specialization,":[27],"its":[28],"direct":[29],"application":[30],"low-capacity":[32],"models":[33],"imposes":[34],"representational":[36],"constraints":[37],"that":[38,85],"limit":[39],"performance":[41,90,101],"gains":[42,91,102,111],"derived":[43],"from":[44],"specialization.":[45],"In":[46],"this":[47],"paper,":[48],"we":[49],"analyze":[50],"these":[51,110],"limitations":[52],"introduce":[54],"Mixture":[56],"of":[57,123],"Task":[58],"Experts":[59],"(MoTE)":[60],"transformer":[61],"block,":[62],"which":[63],"leverages":[64],"task-specialized":[65],"parameters":[66],"trained":[67],"with":[68],"Task-Aware":[69],"Contrastive":[70],"Learning":[71],"(\\tacl)":[72],"enhance":[74],"model":[76],"ability":[77],"generate":[79],"specialized":[80],"embeddings.":[81],"Empirical":[82],"results":[83],"show":[84],"MoTE":[86],"achieves":[87],"$64\\%$":[88],"higher":[89,100],"in":[92],"retrieval":[93],"datasets":[94,105],"($+3.27":[95],"\\rightarrow":[96,107],"+5.21$)":[97],"$43\\%$":[99],"across":[103],"all":[104],"($+1.81":[106],"+2.60$).":[108],"Critically,":[109],"achieved":[113],"without":[114],"altering":[115],"instructions,":[116],"training":[117],"data,":[118],"inference":[119],"time,":[120],"or":[121],"number":[122],"active":[124],"parameters.":[125]},"counts_by_year":[],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}