{"id":"https://openalex.org/W4403577518","doi":"https://doi.org/10.1145/3627673.3679835","title":"Source Prompt: Coordinated Pre-training of Language Models on Diverse Corpora from Multiple Sources","display_name":"Source Prompt: Coordinated Pre-training of Language Models on Diverse Corpora from Multiple Sources","publication_year":2024,"publication_date":"2024-10-20","ids":{"openalex":"https://openalex.org/W4403577518","doi":"https://doi.org/10.1145/3627673.3679835"},"language":"en","primary_location":{"id":"doi:10.1145/3627673.3679835","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3627673.3679835","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050123988","display_name":"Yipei Xu","orcid":"https://orcid.org/0009-0005-8789-2834"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yipei Xu","raw_affiliation_strings":["Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0005-8789-2834","affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064012021","display_name":"Dakuan Lu","orcid":null},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dakuan Lu","raw_affiliation_strings":["Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0006-4937-0805","affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075507821","display_name":"Jiaqing Liang","orcid":"https://orcid.org/0000-0003-0670-5602"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaqing Liang","raw_affiliation_strings":["Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-0670-5602","affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046715682","display_name":"Jin Zhao","orcid":"https://orcid.org/0000-0002-9414-2696"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jin Zhao","raw_affiliation_strings":["Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-9414-2696","affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050775360","display_name":"Xintao Wang","orcid":"https://orcid.org/0000-0003-1428-8677"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xintao Wang","raw_affiliation_strings":["Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-1428-8677","affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082065207","display_name":"Hengkui Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hengkui Wu","raw_affiliation_strings":["Super Symmetry Technology, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0004-9589-0823","affiliations":[{"raw_affiliation_string":"Super Symmetry Technology, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ken Chen","orcid":"https://orcid.org/0009-0009-8353-2523"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ken Chen","raw_affiliation_strings":["Super Symmetry Technology, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0009-8353-2523","affiliations":[{"raw_affiliation_string":"Super Symmetry Technology, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114337278","display_name":"Liujiang Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liujiang Liu","raw_affiliation_strings":["Super Symmetry Technology, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0006-7108-2391","affiliations":[{"raw_affiliation_string":"Super Symmetry Technology, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087215269","display_name":"Yingsi Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yingsi Xin","raw_affiliation_strings":["Super Symmetry Technology, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0004-6460-7458","affiliations":[{"raw_affiliation_string":"Super Symmetry Technology, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061496726","display_name":"Xuepeng Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuepeng Liu","raw_affiliation_strings":["Super Symmetry Technology, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0007-3007-6793","affiliations":[{"raw_affiliation_string":"Super Symmetry Technology, Shanghai, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090455375","display_name":"Yanghua Xiao","orcid":"https://orcid.org/0000-0001-8403-9591"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanghua Xiao","raw_affiliation_strings":["Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-8403-9591","affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065529268","display_name":"Zhixu Li","orcid":"https://orcid.org/0000-0003-2355-288X"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhixu Li","raw_affiliation_strings":["Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-2355-288X","affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2732","last_page":"2741"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9904000163078308,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7705436944961548},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.6241599917411804},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5067384839057922},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4385034143924713},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4339158833026886},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4200686514377594},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.07371503114700317}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7705436944961548},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.6241599917411804},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5067384839057922},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4385034143924713},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4339158833026886},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4200686514377594},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07371503114700317},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3627673.3679835","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3627673.3679835","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7200000286102295,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W1566289585","https://openalex.org/W2109169869","https://openalex.org/W2902689991","https://openalex.org/W2962890089","https://openalex.org/W2970390493","https://openalex.org/W3034640977","https://openalex.org/W3081168214","https://openalex.org/W3095319910","https://openalex.org/W3169113923","https://openalex.org/W3176617251","https://openalex.org/W3177900337","https://openalex.org/W4283761305","https://openalex.org/W4287077733","https://openalex.org/W6767165935"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W3204019825","https://openalex.org/W4394050964","https://openalex.org/W2551249631","https://openalex.org/W4287644835","https://openalex.org/W3092281475","https://openalex.org/W3098003361"],"abstract_inverted_index":{"Pre-trained":[0],"language":[1],"models":[2,34],"(PLMs)":[3],"have":[4],"established":[5],"the":[6,10,20,33,36,70,96,99,104],"new":[7],"paradigm":[8],"in":[9,124],"field":[11],"of":[12,19,32,72,101],"NLP.":[13],"For":[14],"more":[15],"powerful":[16],"PLMs,":[17],"one":[18],"most":[21],"popular":[22],"and":[23,35,106],"successful":[24],"ways":[25],"is":[26],"to":[27],"continuously":[28],"scale":[29],"up":[30],"sizes":[31],"pre-training":[37,79,83,105,114],"corpora.":[38],"These":[39],"large":[40],"corpora,":[41,86],"typically":[42],"obtained":[43],"by":[44],"converging":[45],"smaller":[46],"ones":[47],"from":[48,75],"multiple":[49,76],"sources,":[50],"are":[51],"thus":[52],"growing":[53],"increasingly":[54],"diverse.":[55],"However,":[56],"colossal":[57],"converged":[58],"corpora":[59,74,120],"don't":[60],"always":[61],"enhance":[62],"PLMs'":[63],"performance.":[64],"In":[65],"this":[66],"paper,":[67],"we":[68,87],"identify":[69],"disadvantage":[71],"heterogeneous":[73],"sources":[77],"for":[78],"PLMs.":[80],"Towards":[81],"coordinated":[82],"on":[84,118],"diverse":[85,119],"further":[88],"propose":[89],"Source":[90],"Prompt":[91],"(SP),":[92],"which":[93],"explicitly":[94],"prompt":[95],"model":[97],"with":[98,116],"source":[100],"data":[102],"at":[103],"fine-tuning":[107],"stages.":[108],"Extensive":[109],"experimental":[110],"results":[111],"show":[112],"that":[113],"PLMs":[115],"SP":[117],"significantly":[121],"improves":[122],"performance":[123],"various":[125],"downstream":[126],"tasks.":[127]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}