{"id":"https://openalex.org/W4396882152","doi":"https://doi.org/10.48550/arxiv.2405.06331","title":"LMD3: Language Model Data Density Dependence","display_name":"LMD3: Language Model Data Density Dependence","publication_year":2024,"publication_date":"2024-05-10","ids":{"openalex":"https://openalex.org/W4396882152","doi":"https://doi.org/10.48550/arxiv.2405.06331"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.06331","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.06331","pdf_url":"https://arxiv.org/pdf/2405.06331","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.06331","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025859844","display_name":"John Kirchenbauer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kirchenbauer, John","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047062065","display_name":"Garrett Honke","orcid":"https://orcid.org/0000-0001-6678-044X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Honke, Garrett","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078715414","display_name":"Gowthami Somepalli","orcid":"https://orcid.org/0000-0002-5380-1413"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Somepalli, Gowthami","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049400969","display_name":"Jonas Geiping","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Geiping, Jonas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098060238","display_name":"Daphne Ippolito","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ippolito, Daphne","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101561897","display_name":"Katherine Lee","orcid":"https://orcid.org/0000-0002-9537-6195"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Katherine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060687985","display_name":"Tom Goldstein","orcid":"https://orcid.org/0000-0003-1660-9307"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Goldstein, Tom","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101143336","display_name":"David Andr\u00e9","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Andre, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.988099992275238,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.988099992275238,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9771000146865845,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9251999855041504,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.3637378513813019},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.25506794452667236}],"concepts":[{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3637378513813019},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.25506794452667236}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.06331","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.06331","pdf_url":"https://arxiv.org/pdf/2405.06331","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.06331","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.06331","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.06331","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.06331","pdf_url":"https://arxiv.org/pdf/2405.06331","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4396882152.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2054730750","https://openalex.org/W3200949300","https://openalex.org/W3044993392","https://openalex.org/W1550105798","https://openalex.org/W2964500046","https://openalex.org/W4361004270","https://openalex.org/W3123681204","https://openalex.org/W1497930768","https://openalex.org/W2752702163","https://openalex.org/W2112365323"],"abstract_inverted_index":{"We":[0,86],"develop":[1],"a":[2,25,46,54,74,99,127],"methodology":[3],"for":[4,40,126],"analyzing":[5],"language":[6],"model":[7,81],"task":[8],"performance":[9,59],"at":[10],"the":[11,34,37,58,63,78,96,117,123],"individual":[12],"example":[13],"level":[14],"based":[15],"on":[16,28,103],"training":[17,38,107,124],"data":[18,30,68,125],"density":[19,84],"estimation.":[20],"Experiments":[21,65],"with":[22,66],"paraphrasing":[23],"as":[24],"controlled":[26],"intervention":[27],"finetuning":[29],"demonstrate":[31,69],"that":[32,70,88],"increasing":[33],"support":[35,118],"in":[36,45,49,80,122],"distribution":[39],"specific":[41],"test":[42,129],"queries":[43],"results":[44],"measurable":[47],"increase":[48,60],"density,":[50],"which":[51],"is":[52],"also":[53],"significant":[55,75],"predictor":[56],"of":[57,77,95,98,105],"caused":[61],"by":[62],"intervention.":[64],"pretraining":[67],"we":[71],"can":[72,91,110],"explain":[73],"fraction":[76],"variance":[79],"perplexity":[82],"via":[83],"measurements.":[85],"conclude":[87],"our":[89],"framework":[90],"provide":[92],"statistical":[93],"evidence":[94],"dependence":[97],"target":[100],"model's":[101],"predictions":[102],"subsets":[104],"its":[106],"data,":[108],"and":[109],"more":[111],"generally":[112],"be":[113],"used":[114],"to":[115],"characterize":[116],"(or":[119],"lack":[120],"thereof)":[121],"given":[128],"task.":[130]},"counts_by_year":[],"updated_date":"2026-08-11T07:18:39.950985","created_date":"2025-10-10T00:00:00"}