{"id":"https://openalex.org/W4403578767","doi":"https://doi.org/10.48550/arxiv.2410.12881","title":"MIND: Math Informed syNthetic Dialogues for Pretraining LLMs","display_name":"MIND: Math Informed syNthetic Dialogues for Pretraining LLMs","publication_year":2024,"publication_date":"2024-10-15","ids":{"openalex":"https://openalex.org/W4403578767","doi":"https://doi.org/10.48550/arxiv.2410.12881"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.12881","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.12881","pdf_url":"https://arxiv.org/pdf/2410.12881","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.12881","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112924096","display_name":"Syeda Nahida Akter","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Akter, Syeda Nahida","raw_affiliation_strings":[", Carnegie Mellon University 2 , Boston University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":", Carnegie Mellon University 2 , Boston University","institution_ids":["https://openalex.org/I111088046","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000188096","display_name":"Shrimai Prabhumoye","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Prabhumoye, Shrimai","raw_affiliation_strings":[", Carnegie Mellon University 2 , Boston University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":", Carnegie Mellon University 2 , Boston University","institution_ids":["https://openalex.org/I111088046","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Kamalu, John","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kamalu, John","raw_affiliation_strings":[", Carnegie Mellon University 2 , Boston University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":", Carnegie Mellon University 2 , Boston University","institution_ids":["https://openalex.org/I111088046","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090158590","display_name":"Sanjeev Satheesh","orcid":"https://orcid.org/0009-0008-5139-1036","observed_orcids":["https://orcid.org/0009-0008-5139-1036"]},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Satheesh, Sanjeev","raw_affiliation_strings":[", Carnegie Mellon University 2 , Boston University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":", Carnegie Mellon University 2 , Boston University","institution_ids":["https://openalex.org/I111088046","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063241234","display_name":"Eric Nyberg","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nyberg, Eric","raw_affiliation_strings":[", Carnegie Mellon University 2 , Boston University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":", Carnegie Mellon University 2 , Boston University","institution_ids":["https://openalex.org/I111088046","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031170568","display_name":"Mostofa Ali Patwary","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Patwary, Mostofa","raw_affiliation_strings":[", Carnegie Mellon University 2 , Boston University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":", Carnegie Mellon University 2 , Boston University","institution_ids":["https://openalex.org/I111088046","https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072436307","display_name":"Mohammad Shoeybi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shoeybi, Mohammad","raw_affiliation_strings":[", Carnegie Mellon University 2 , Boston University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":", Carnegie Mellon University 2 , Boston University","institution_ids":["https://openalex.org/I111088046","https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066242985","display_name":"Bryan Catanzaro","orcid":"https://orcid.org/0000-0003-0034-7728","observed_orcids":["https://orcid.org/0000-0003-0034-7728"]},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Catanzaro, Bryan","raw_affiliation_strings":[", Carnegie Mellon University 2 , Boston University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":", Carnegie Mellon University 2 , Boston University","institution_ids":["https://openalex.org/I111088046","https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8032000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8032000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.05480000004172325,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03020000085234642,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/synthetic-dialogue-generation","display_name":"synthetic dialogue generation","score":0.8370000123977661},{"id":"https://openalex.org/keywords/large-language-model-pretraining","display_name":"large language model pretraining","score":0.8149999976158142},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.9929999709129333},{"id":"https://openalex.org/keywords/synthetic-data","display_name":"synthetic data","score":0.5649999976158142},{"id":"https://openalex.org/keywords/multi-hop-reasoning","display_name":"multi-hop reasoning","score":0.7519999742507935}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3804498314857483},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.37597405910491943}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.12881","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.12881","pdf_url":"https://arxiv.org/pdf/2410.12881","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.12881","doi":"https://doi.org/10.48550/arxiv.2410.12881","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.12881","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.12881","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.12881","pdf_url":"https://arxiv.org/pdf/2410.12881","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Quality education","id":"https://metadata.un.org/sdg/4","score":0.4156000018119812}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403578767.pdf","grobid_xml":"https://content.openalex.org/works/W4403578767.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4415799149","https://openalex.org/W4416017259","https://openalex.org/W7077674324","https://openalex.org/W7166817036","https://openalex.org/W7106237852","https://openalex.org/W4392222438","https://openalex.org/W4415798372","https://openalex.org/W4416035588","https://openalex.org/W4403928588","https://openalex.org/W4416246779"],"abstract_inverted_index":{"The":[0],"utility":[1],"of":[2,77],"synthetic":[3,41,84,128],"data":[4,8,42,131,147],"to":[5,12,45,49,124,134,144,154],"enhance":[6],"pretraining":[7,133,155],"quality":[9],"and":[10,35,62,126,129,185],"hence":[11],"improve":[13],"downstream":[14],"task":[15],"accuracy":[16],"has":[17],"been":[18],"widely":[19],"explored":[20],"in":[21,32,91,138,168,178],"recent":[22],"large":[23],"language":[24],"models":[25],"(LLMs).":[26],"Yet,":[27],"these":[28],"approaches":[29],"fall":[30],"inadequate":[31],"complex,":[33],"multi-hop":[34],"mathematical":[36,74,139,169],"reasoning":[37,75,170,188],"tasks":[38,189],"as":[39],"the":[40,50,73,136,142],"typically":[43],"fails":[44],"add":[46],"complementary":[47],"knowledge":[48,106,180],"existing":[51],"raw":[52,130,146,158],"corpus.":[53],"In":[54],"this":[55],"work,":[56],"we":[57,82],"propose":[58],"a":[59,92,160],"novel":[60],"large-scale":[61],"diverse":[63],"Math":[64],"Informed":[65],"syNthetic":[66],"Dialogue":[67],"(MIND)":[68],"generation":[69],"method":[70],"that":[71,104],"improves":[72],"ability":[76],"LLMs.":[78],"Specifically,":[79],"using":[80],"MIND,":[81],"generate":[83],"conversations":[85],"based":[86],"on":[87,157,163],"OpenWebMath":[88],"(OWM),":[89],"resulting":[90],"new":[93],"math":[94,116],"corpus,":[95],"MIND-OWM.":[96],"Our":[97],"experiments":[98],"with":[99],"different":[100],"conversational":[101],"settings":[102],"reveal":[103],"incorporating":[105],"gaps":[107],"between":[108],"dialog":[109],"participants":[110],"is":[111],"essential":[112],"for":[113],"generating":[114],"high-quality":[115],"data.":[117],"We":[118],"further":[119],"identify":[120],"an":[121],"effective":[122],"way":[123],"format":[125],"integrate":[127],"during":[132],"maximize":[135],"gain":[137],"reasoning,":[140],"emphasizing":[141],"need":[143],"restructure":[145],"rather":[148],"than":[149],"use":[150],"it":[151],"as-is.":[152],"Compared":[153],"just":[156],"data,":[159],"model":[161],"pretrained":[162],"MIND-OWM":[164],"shows":[165],"significant":[166],"boost":[167],"(GSM8K:":[171],"+13.42%,":[172],"MATH:":[173],"+2.30%),":[174],"including":[175],"superior":[176],"performance":[177],"specialized":[179],"(MMLU:":[181],"+4.55%,":[182],"MMLU-STEM:":[183],"+4.28%)":[184],"general":[186],"purpose":[187],"(GENERAL":[190],"REASONING:":[191],"+2.51%).":[192]},"counts_by_year":[],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2024-10-20T00:00:00"}