{"id":"https://openalex.org/W4394973043","doi":"https://doi.org/10.48550/arxiv.2404.11706","title":"Pretraining Billion-scale Geospatial Foundational Models on Frontier","display_name":"Pretraining Billion-scale Geospatial Foundational Models on Frontier","publication_year":2024,"publication_date":"2024-04-17","ids":{"openalex":"https://openalex.org/W4394973043","doi":"https://doi.org/10.48550/arxiv.2404.11706"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2404.11706","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.11706","pdf_url":"https://arxiv.org/pdf/2404.11706","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2404.11706","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006493183","display_name":"Aristeidis Tsaris","orcid":"https://orcid.org/0000-0002-7734-3349"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tsaris, Aristeidis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034044972","display_name":"Philipe Dias","orcid":"https://orcid.org/0000-0001-9427-7112"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dias, Philipe Ambrozio","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089748744","display_name":"Abhishek Potnis","orcid":"https://orcid.org/0000-0001-8168-857X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Potnis, Abhishek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051882926","display_name":"Junqi Yin","orcid":"https://orcid.org/0000-0003-3843-5520"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Junqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101916963","display_name":"Feiyi Wang","orcid":"https://orcid.org/0000-0002-0099-1559"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Feiyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5083948807","display_name":"Dalton Lunga","orcid":"https://orcid.org/0000-0003-0054-1141"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lunga, Dalton","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10757","display_name":"Geographic Information Systems Studies","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/3305","display_name":"Geography, Planning and Development"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10757","display_name":"Geographic Information Systems Studies","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/3305","display_name":"Geography, Planning and Development"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11106","display_name":"Data Management and Algorithms","score":0.984499990940094,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9319000244140625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/frontier","display_name":"Frontier","score":0.8330502510070801},{"id":"https://openalex.org/keywords/geospatial-analysis","display_name":"Geospatial analysis","score":0.8300195932388306},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.6259013414382935},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.571816086769104},{"id":"https://openalex.org/keywords/regional-science","display_name":"Regional science","score":0.4192880094051361},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.319265216588974},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3161226511001587},{"id":"https://openalex.org/keywords/archaeology","display_name":"Archaeology","score":0.1892739236354828},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.18102332949638367}],"concepts":[{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.8330502510070801},{"id":"https://openalex.org/C9770341","wikidata":"https://www.wikidata.org/wiki/Q1938983","display_name":"Geospatial analysis","level":2,"score":0.8300195932388306},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.6259013414382935},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.571816086769104},{"id":"https://openalex.org/C148383697","wikidata":"https://www.wikidata.org/wiki/Q1781695","display_name":"Regional science","level":1,"score":0.4192880094051361},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.319265216588974},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3161226511001587},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.1892739236354828},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.18102332949638367}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2404.11706","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.11706","pdf_url":"https://arxiv.org/pdf/2404.11706","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2404.11706","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2404.11706","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2404.11706","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.11706","pdf_url":"https://arxiv.org/pdf/2404.11706","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1026679911","display_name":null,"funder_award_id":"DE-AC05","funder_id":"https://openalex.org/F4320316892","funder_display_name":"UT-Battelle"},{"id":"https://openalex.org/G121915436","display_name":null,"funder_award_id":"DEAC05- 00OR22725","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G1645119126","display_name":null,"funder_award_id":"AC05-00OR22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G166562317","display_name":null,"funder_award_id":"DE-AC05- 00OR22725","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"},{"id":"https://openalex.org/G203625172","display_name":null,"funder_award_id":"DE--AC05--00OR22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G2988778103","display_name":null,"funder_award_id":"DE-AC05?","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G313676650","display_name":null,"funder_award_id":"DEAC05-00OR22725","funder_id":"https://openalex.org/F4320316892","funder_display_name":"UT-Battelle"},{"id":"https://openalex.org/G5726405315","display_name":null,"funder_award_id":"DE-AC05","funder_id":"https://openalex.org/F4320306250","funder_display_name":"Battelle"},{"id":"https://openalex.org/G8269158468","display_name":null,"funder_award_id":"AC05-00OR22725","funder_id":"https://openalex.org/F4320332359","funder_display_name":"Office of Science"},{"id":"https://openalex.org/G8906985441","display_name":null,"funder_award_id":"00OR22725","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G8943143067","display_name":null,"funder_award_id":"AC05-00OR22725","funder_id":"https://openalex.org/F4320316892","funder_display_name":"UT-Battelle"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320306250","display_name":"Battelle","ror":"https://ror.org/01h5tnr73"},{"id":"https://openalex.org/F4320316892","display_name":"UT-Battelle","ror":"https://ror.org/04nza6677"},{"id":"https://openalex.org/F4320332359","display_name":"Office of Science","ror":"https://ror.org/00mmn6b08"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394973043.pdf","grobid_xml":"https://content.openalex.org/works/W4394973043.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2347401120","https://openalex.org/W2261902776","https://openalex.org/W2041961361","https://openalex.org/W4280507437","https://openalex.org/W3118689035","https://openalex.org/W2553922746","https://openalex.org/W3048452028","https://openalex.org/W2001703222","https://openalex.org/W2331360809","https://openalex.org/W4379390828"],"abstract_inverted_index":{"As":[0],"AI":[1],"workloads":[2],"increase":[3],"in":[4,59,103,112,153,172],"scope,":[5],"generalization":[6],"capability":[7],"becomes":[8],"challenging":[9],"for":[10,17,69,136,225,259],"small":[11],"task-specific":[12],"models":[13,82,227,258],"and":[14,40,63,110,132,151,201,237],"their":[15],"demand":[16],"large":[18,53,85,257],"amounts":[19],"of":[20,98,108,216],"labeled":[21],"training":[22,134],"samples":[23],"increases.":[24],"On":[25],"the":[26,149,154,158,189,217],"contrary,":[27],"Foundation":[28],"Models":[29],"(FMs)":[30],"are":[31,106],"trained":[32],"with":[33,49,89,228],"internet-scale":[34],"unlabeled":[35],"data":[36,99,116,202],"via":[37],"self-supervised":[38],"learning":[39],"have":[41,55,72],"been":[42,73],"shown":[43],"to":[44,46,75,124,169,231,249],"adapt":[45],"various":[47],"tasks":[48],"minimal":[50],"fine-tuning.":[51],"Although":[52],"FMs":[54,68,131],"demonstrated":[56],"significant":[57],"impact":[58,152],"natural":[60],"language":[61],"processing":[62],"computer":[64],"vision,":[65],"efforts":[66],"toward":[67],"geospatial":[70,115,137,260],"applications":[71,138],"restricted":[74],"smaller":[76],"size":[77,165,229],"models,":[78],"as":[79],"pretraining":[80,140],"larger":[81,162],"requires":[83],"very":[84],"computing":[86],"resources":[87,254],"equipped":[88],"state-of-the-art":[90],"hardware":[91],"accelerators.":[92],"Current":[93],"satellite":[94],"constellations":[95],"collect":[96],"100+TBs":[97],"a":[100,179],"day,":[101],"resulting":[102],"images":[104],"that":[105],"billions":[107],"pixels":[109],"multimodal":[111],"nature.":[113],"Such":[114],"poses":[117],"unique":[118],"challenges":[119],"opening":[120],"up":[121,168,230],"new":[122],"opportunities":[123],"develop":[125],"FMs.":[126],"We":[127,145],"investigate":[128],"billion":[129],"scale":[130],"HPC":[133,253],"profiles":[135],"by":[139,156],"on":[141,188,247],"publicly":[142],"available":[143],"data.":[144],"studied":[146],"from":[147],"end-to-end":[148],"performance":[150,186,223,238],"solution":[155],"scaling":[157],"model":[159,166,200],"size.":[160],"Our":[161],"3B":[163],"parameter":[164,181],"achieves":[167],"30%":[170],"improvement":[171],"top1":[173],"scene":[174],"classification":[175],"accuracy":[176],"when":[177,255],"comparing":[178],"100M":[180],"model.":[182],"Moreover,":[183],"we":[184,197,213,244],"detail":[185],"experiments":[187],"Frontier":[190],"supercomputer,":[191],"America's":[192],"first":[193],"exascale":[194],"system,":[195],"where":[196],"study":[198,214],"different":[199,241],"parallel":[203],"approaches":[204],"using":[205],"PyTorch's":[206],"Fully":[207],"Sharded":[208],"Data":[209],"Parallel":[210],"library.":[211],"Specifically,":[212],"variants":[215],"Vision":[218],"Transformer":[219],"architecture":[220],"(ViT),":[221],"conducting":[222],"analysis":[224],"ViT":[226],"15B":[232],"parameters.":[233],"By":[234],"discussing":[235],"throughput":[236],"bottlenecks":[239],"under":[240],"parallelism":[242],"configurations,":[243],"offer":[245],"insights":[246],"how":[248],"leverage":[250],"such":[251],"leadership-class":[252],"developing":[256],"imagery":[261],"applications.":[262]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-09-08T07:41:39.490200","created_date":"2025-10-10T00:00:00"}