{"id":"https://openalex.org/W4392019361","doi":"https://doi.org/10.48550/arxiv.2402.13064","title":"Synthetic Data (Almost) from Scratch: Generalized Instruction Tuning for Language Models","display_name":"Synthetic Data (Almost) from Scratch: Generalized Instruction Tuning for Language Models","publication_year":2024,"publication_date":"2024-02-20","ids":{"openalex":"https://openalex.org/W4392019361","doi":"https://doi.org/10.48550/arxiv.2402.13064"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.13064","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.13064","pdf_url":"https://arxiv.org/pdf/2402.13064","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.13064","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100327732","display_name":"Haoran Li","orcid":"https://orcid.org/0000-0002-5994-0203","observed_orcids":["https://orcid.org/0000-0002-5994-0203"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Haoran","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113750837","display_name":"Qingxiu Dong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Qingxiu","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109434969","display_name":"Zhengyang Tang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Zhengyang","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003237783","display_name":"Chaojun Wang","orcid":"https://orcid.org/0000-0002-1632-1728","observed_orcids":["https://orcid.org/0000-0002-1632-1728"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Chaojun","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100343569","display_name":"Xingxing Zhang","orcid":"https://orcid.org/0000-0003-4012-3796","observed_orcids":["https://orcid.org/0000-0003-4012-3796"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Xingxing","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020769158","display_name":"Haoyang Huang","orcid":"https://orcid.org/0009-0009-1431-9112","observed_orcids":["https://orcid.org/0009-0009-1431-9112","https://orcid.org/0000-0002-3804-5437"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Haoyang","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061624006","display_name":"Shaohan Huang","orcid":"https://orcid.org/0000-0003-4324-6337","observed_orcids":["https://orcid.org/0000-0003-4324-6337"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Shaohan","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022782194","display_name":"Xiaolong Huang","orcid":"https://orcid.org/0000-0001-6253-433X","observed_orcids":["https://orcid.org/0000-0001-6253-433X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Xiaolong","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Huang, Zeqiang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zeqiang","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100416386","display_name":"Dongdong Zhang","orcid":"https://orcid.org/0000-0003-0833-7903","observed_orcids":["https://orcid.org/0000-0003-0833-7903"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Dongdong","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012938939","display_name":"Yuxian Gu","orcid":"https://orcid.org/0000-0002-4607-7025","observed_orcids":["https://orcid.org/0000-0002-4607-7025"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Yuxian","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Cheng, Xin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Xin","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113162264","display_name":"Xun Wang","orcid":"https://orcid.org/0009-0008-8140-2621","observed_orcids":["https://orcid.org/0009-0008-8140-2621"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xun","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107977473","display_name":"Siqing Chen","orcid":"https://orcid.org/0000-0001-8530-4174","observed_orcids":["https://orcid.org/0000-0001-8530-4174"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Si-Qing","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103661051","display_name":"Li Mei Dong","orcid":"https://orcid.org/0009-0008-9750-5596","observed_orcids":["https://orcid.org/0009-0008-9750-5596"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Li","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005529387","display_name":"Wei L\u00fc","orcid":"https://orcid.org/0000-0001-6769-2695","observed_orcids":["https://orcid.org/0000-0001-6769-2695"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Wei","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110285832","display_name":"Zhifang Sui","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sui, Zhifang","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057282504","display_name":"Benyou Wang","orcid":"https://orcid.org/0000-0002-1501-9914","observed_orcids":["https://orcid.org/0000-0002-1501-9914","https://orcid.org/0009-0000-3606-1364"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Benyou","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Lam, Wai","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lam, Wai","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014662947","display_name":"Furu Wei","orcid":"https://orcid.org/0000-0002-7810-5852","observed_orcids":["https://orcid.org/0000-0002-7810-5852"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Furu","raw_affiliation_strings":["https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"https ://aka. / GeneralAI","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9067000150680542,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9067000150680542,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.03700000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.026399999856948853,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/instruction-tuning","display_name":"instruction tuning","score":0.9350000023841858},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.890999972820282},{"id":"https://openalex.org/keywords/knowledge-taxonomy","display_name":"knowledge taxonomy","score":0.6769999861717224},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.5080000162124634},{"id":"https://openalex.org/keywords/coding","display_name":"coding","score":0.5709999799728394},{"id":"https://openalex.org/keywords/logical-reasoning","display_name":"logical reasoning","score":0.9440000057220459},{"id":"https://openalex.org/keywords/instruction-following","display_name":"instruction following","score":0.8309999704360962},{"id":"https://openalex.org/keywords/mistral","display_name":"Mistral","score":0.3580000102519989}],"concepts":[{"id":"https://openalex.org/C2781235140","wikidata":"https://www.wikidata.org/wiki/Q275131","display_name":"Scratch","level":2,"score":0.9126306772232056},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6322176456451416},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4447154998779297},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.36225706338882446},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29162704944610596}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.13064","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.13064","pdf_url":"https://arxiv.org/pdf/2402.13064","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.13064","doi":"https://doi.org/10.48550/arxiv.2402.13064","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.13064","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.13064","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.13064","pdf_url":"https://arxiv.org/pdf/2402.13064","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.6164000034332275,"id":"https://metadata.un.org/sdg/4","display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392019361.pdf","grobid_xml":"https://content.openalex.org/works/W4392019361.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7140140937","https://openalex.org/W6948249650","https://openalex.org/W4387947646","https://openalex.org/W4389518761","https://openalex.org/W4412944565","https://openalex.org/W4401042743","https://openalex.org/W7071957046","https://openalex.org/W4385570984","https://openalex.org/W4312091890","https://openalex.org/W7126207509"],"abstract_inverted_index":{"We":[0],"introduce":[1],"Generalized":[2],"Instruction":[3],"Tuning":[4],"(called":[5],"GLAN),":[6],"a":[7,39,93,105,135,200],"general":[8,171],"and":[9,45,49,76,82,101,145,189],"scalable":[10],"method":[11],"for":[12,98,186],"instruction":[13,33,53,172],"tuning":[14,34],"of":[15,42,96,124,142,179],"Large":[16],"Language":[17],"Models":[18],"(LLMs).":[19],"Unlike":[20],"prior":[21],"work":[22],"that":[23,156],"relies":[24],"on":[25,149],"seed":[26],"examples":[27],"or":[28,192],"existing":[29],"datasets":[30],"to":[31,78,103,108,130,170],"construct":[32],"data,":[35],"GLAN":[36,157,184],"exclusively":[37],"utilizes":[38],"pre-curated":[40],"taxonomy":[41,71],"human":[43,65,74,143],"knowledge":[44,75,144],"capabilities":[46,77],"as":[47],"input":[48],"generates":[50],"large-scale":[51],"synthetic":[52],"data":[54,178],"across":[55,138],"all":[56],"disciplines.":[57],"Specifically,":[58],"inspired":[59],"by":[60,72,88,197],"the":[61,70,115,125,139],"systematic":[62],"structure":[63],"in":[64,120,159],"education":[66],"system,":[67],"we":[68,91,127],"build":[69],"decomposing":[73],"various":[79],"fields,":[80],"sub-fields":[81],"ultimately,":[83],"distinct":[84],"disciplines":[85],"semi-automatically,":[86],"facilitated":[87],"LLMs.":[89,113],"Subsequently,":[90],"generate":[92,131],"comprehensive":[94],"list":[95],"subjects":[97],"every":[99,121],"discipline":[100],"proceed":[102],"design":[104],"syllabus":[106],"tailored":[107],"each":[109],"subject,":[110],"again":[111],"utilizing":[112],"With":[114],"fine-grained":[116],"key":[117],"concepts":[118],"detailed":[119],"class":[122],"session":[123],"syllabus,":[126],"are":[128],"able":[129],"diverse":[132],"instructions":[133],"with":[134],"broad":[136],"coverage":[137],"entire":[140],"spectrum":[141],"skills.":[146],"Extensive":[147],"experiments":[148],"large":[150],"language":[151],"models":[152],"(e.g.,":[153],"Mistral)":[154],"demonstrate":[155],"excels":[158],"multiple":[160],"dimensions":[161],"from":[162],"mathematical":[163],"reasoning,":[164],"coding,":[165],"academic":[166],"exams,":[167],"logical":[168],"reasoning":[169],"following":[173],"without":[174],"using":[175],"task-specific":[176],"training":[177],"these":[180],"tasks.":[181],"In":[182],"addition,":[183],"allows":[185],"easy":[187],"customization":[188],"new":[190,201],"fields":[191],"skills":[193],"can":[194],"be":[195],"added":[196],"simply":[198],"incorporating":[199],"node":[202],"into":[203],"our":[204],"taxonomy.":[205]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2}],"updated_date":"2026-10-13T10:08:12.033832","created_date":"2024-02-22T00:00:00"}