{"id":"https://openalex.org/W4415277262","doi":"https://doi.org/10.48550/arxiv.2510.13697","title":"On Pretraining for Project-Level Code Completion","display_name":"On Pretraining for Project-Level Code Completion","publication_year":2025,"publication_date":"2025-10-15","ids":{"openalex":"https://openalex.org/W4415277262","doi":"https://doi.org/10.48550/arxiv.2510.13697"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.13697","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.13697","pdf_url":"https://arxiv.org/pdf/2510.13697","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.13697","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Sapronov, Maksim","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sapronov, Maksim","raw_affiliation_strings":["JetBrains Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"JetBrains Research","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023869257","display_name":"Evgeniy Vladimirovich Glukhov","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Glukhov, Evgeniy","raw_affiliation_strings":["JetBrains Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"JetBrains Research","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-completion","display_name":"code completion","score":0.8790000081062317},{"id":"https://openalex.org/keywords/large-language-models-for-code","display_name":"code large language models","score":0.7919999957084656},{"id":"https://openalex.org/keywords/in-context-learning","display_name":"in-context learning","score":0.8529999852180481},{"id":"https://openalex.org/keywords/context-window-extension","display_name":"context window extension","score":0.5720000267028809}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7638000249862671},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6897000074386597},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6628000140190125},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.5823000073432922},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5224000215530396},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.4810999929904938},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4496999979019165},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.4065999984741211},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.39329999685287476},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3855000138282776},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.35109999775886536},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33809998631477356},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.32600000500679016},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32179999351501465},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3172000050544739},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.28299999237060547},{"id":"https://openalex.org/C2988963302","wikidata":"https://www.wikidata.org/wiki/Q629206","display_name":"Program code","level":2,"score":0.2750999927520752},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.273499995470047},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.2721000015735626},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.27140000462532043},{"id":"https://openalex.org/C135598885","wikidata":"https://www.wikidata.org/wiki/Q1366302","display_name":"Row","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2572999894618988}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.13697","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.13697","pdf_url":"https://arxiv.org/pdf/2510.13697","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.13697","doi":"https://doi.org/10.48550/arxiv.2510.13697","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.13697","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.13697","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.13697","pdf_url":"https://arxiv.org/pdf/2510.13697","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415277262.pdf","grobid_xml":"https://content.openalex.org/works/W4415277262.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W3126675481","https://openalex.org/W4379548473","https://openalex.org/W4382239850","https://openalex.org/W4382334332","https://openalex.org/W4382603041","https://openalex.org/W4384648639","https://openalex.org/W4386185625","https://openalex.org/W4386435778","https://openalex.org/W4387356039","https://openalex.org/W4387559790","https://openalex.org/W4387561453","https://openalex.org/W4391272793","https://openalex.org/W4399795128","https://openalex.org/W4399795308","https://openalex.org/W4399912658","https://openalex.org/W4401042382","https://openalex.org/W4403796339","https://openalex.org/W4412889664","https://openalex.org/W6967084484","https://openalex.org/W7125135101","https://openalex.org/W7131800956"],"related_works":["https://openalex.org/W4381571810","https://openalex.org/W4409362724","https://openalex.org/W4414903186","https://openalex.org/W4399401100","https://openalex.org/W4360834937","https://openalex.org/W7125960116","https://openalex.org/W4411552869","https://openalex.org/W7126001353","https://openalex.org/W4402671782","https://openalex.org/W4405301098"],"abstract_inverted_index":{"Repository-level":[0],"pretraining":[1],"is":[2],"commonly":[3],"used":[4],"to":[5,12,20,51,109,141],"enable":[6],"large":[7],"language":[8],"models":[9,72],"for":[10],"code":[11,25,138],"leverage":[13],"codebase-wide":[14],"context.":[15],"This":[16],"enhances":[17],"their":[18],"ability":[19],"generate":[21],"accurate":[22],"and":[23,147],"context-aware":[24],"completions.":[26],"In":[27],"this":[28],"work,":[29],"we":[30,119],"investigate":[31],"how":[32],"different":[33],"repository-processing":[34,96],"strategies":[35],"affect":[36],"in-context":[37],"learning":[38],"in":[39],"OpenCoder,":[40],"a":[41,67,110,122],"1.5B-parameter":[42],"model.":[43],"We":[44,92],"extend":[45],"its":[46],"context":[47],"window":[48],"from":[49,107],"4,096":[50],"16,384":[52],"tokens":[53,59],"by":[54],"training":[55,125],"on":[56,66,86],"additional":[57],"1B":[58],"of":[60,77,79],"curated":[61],"repository-level":[62,137],"data.":[63],"Despite":[64],"relying":[65],"smaller":[68],"dataset":[69],"than":[70],"competing":[71],"(which":[73],"often":[74],"use":[75],"hundreds":[76],"billions":[78],"tokens),":[80],"our":[81],"model":[82],"achieves":[83],"comparable":[84],"performance":[85],"the":[87,103,128],"Long":[88],"Code":[89],"Arena":[90],"benchmark.":[91],"find":[93],"that":[94,121],"various":[95],"techniques":[97],"yield":[98],"similarly":[99],"strong":[100],"results,":[101],"with":[102,143],"primary":[104],"gain":[105],"coming":[106],"adapting":[108],"new":[111],"rotary":[112],"positional":[113],"embedding":[114],"(RoPE)":[115],"scaling":[116],"parameter.":[117],"Finally,":[118],"show":[120],"simpler":[123],"file-level":[124],"approach":[126],"at":[127],"original":[129],"sequence":[130],"length":[131],"remains":[132],"highly":[133],"effective,":[134],"opening":[135],"up":[136],"completion":[139],"research":[140],"settings":[142],"more":[144],"constrained":[145],"data":[146],"compute":[148],"resources.":[149]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-17T00:00:00"}