{"id":"https://openalex.org/W4403258217","doi":"https://doi.org/10.1177/10943420241288242","title":"Employing artificial intelligence to steer exascale workflows with colmena","display_name":"Employing artificial intelligence to steer exascale workflows with colmena","publication_year":2024,"publication_date":"2024-10-08","ids":{"openalex":"https://openalex.org/W4403258217","doi":"https://doi.org/10.1177/10943420241288242"},"language":"en","primary_location":{"id":"doi:10.1177/10943420241288242","is_oa":false,"landing_page_url":"https://doi.org/10.1177/10943420241288242","pdf_url":null,"source":{"id":"https://openalex.org/S60606485","display_name":"The International Journal of High Performance Computing Applications","issn_l":"1094-3420","issn":["1094-3420","1741-2846"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The International Journal of High Performance Computing Applications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.osti.gov/biblio/2461626","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021403239","display_name":"Logan Ward","orcid":"https://orcid.org/0000-0002-1323-5939"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Logan Ward","raw_affiliation_strings":["Data Science and Learning Division, Argonne National Laboratory, Argonne, IL, USA"],"affiliations":[{"raw_affiliation_string":"Data Science and Learning Division, Argonne National Laboratory, Argonne, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069913075","display_name":"J. Gregory Pauloski","orcid":"https://orcid.org/0000-0002-6547-6902"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"J. Gregory Pauloski","raw_affiliation_strings":["Department of Computer Science, University of Chicago, Chicago, IL, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108154004","display_name":"Val\u00e9rie Hayot\u2010Sasson","orcid":null},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Valerie Hayot-Sasson","raw_affiliation_strings":["Department of Computer Science, University of Chicago, Chicago, IL, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001788942","display_name":"Yadu Babuji","orcid":"https://orcid.org/0000-0002-9162-6003"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yadu Babuji","raw_affiliation_strings":["Department of Computer Science, University of Chicago, Chicago, IL, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059741659","display_name":"Alexander Brace","orcid":"https://orcid.org/0000-0001-9873-9177"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander Brace","raw_affiliation_strings":["Department of Computer Science, University of Chicago, Chicago, IL, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088145676","display_name":"Ryan Chard","orcid":"https://orcid.org/0000-0002-6781-7432"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ryan Chard","raw_affiliation_strings":["Data Science and Learning Division, Argonne National Laboratory, Argonne, IL, USA"],"affiliations":[{"raw_affiliation_string":"Data Science and Learning Division, Argonne National Laboratory, Argonne, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065464552","display_name":"Kyle Chard","orcid":"https://orcid.org/0000-0002-7370-4805"},"institutions":[{"id":"https://openalex.org/I40347166","display_name":"University of Chicago","ror":"https://ror.org/024mw5h28","country_code":"US","type":"education","lineage":["https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kyle Chard","raw_affiliation_strings":["Department of Computer Science, University of Chicago, Chicago, IL, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Chicago, Chicago, IL, USA","institution_ids":["https://openalex.org/I40347166"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014920685","display_name":"Rajeev Thakur","orcid":"https://orcid.org/0000-0002-5532-3048"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rajeev Thakur","raw_affiliation_strings":["Data Science and Learning Division, Argonne National Laboratory, Argonne, IL, USA"],"affiliations":[{"raw_affiliation_string":"Data Science and Learning Division, Argonne National Laboratory, Argonne, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032231503","display_name":"Ian Foster","orcid":"https://orcid.org/0000-0003-2129-5269"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ian Foster","raw_affiliation_strings":["Data Science and Learning Division, Argonne National Laboratory, Argonne, IL, USA"],"affiliations":[{"raw_affiliation_string":"Data Science and Learning Division, Argonne National Laboratory, Argonne, IL, USA","institution_ids":["https://openalex.org/I1282105669"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5021403239"],"corresponding_institution_ids":["https://openalex.org/I1282105669"],"apc_list":null,"apc_paid":null,"fwci":3.9929,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.94745266,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"39","issue":"1","first_page":"52","last_page":"64"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9857000112533569,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.8794058561325073},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8409163355827332},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.7082961201667786},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.5829581022262573},{"id":"https://openalex.org/keywords/exascale-computing","display_name":"Exascale computing","score":0.5587871670722961},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.5523199439048767},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.47527092695236206},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.45740416646003723},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3726311922073364},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.35200557112693787},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.34265661239624023},{"id":"https://openalex.org/keywords/parallel-computing","display_name":"Parallel computing","score":0.2466573715209961},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.14757508039474487},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.1152726411819458}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.8794058561325073},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8409163355827332},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.7082961201667786},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5829581022262573},{"id":"https://openalex.org/C2778837361","wikidata":"https://www.wikidata.org/wiki/Q2450880","display_name":"Exascale computing","level":3,"score":0.5587871670722961},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.5523199439048767},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.47527092695236206},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.45740416646003723},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3726311922073364},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35200557112693787},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.34265661239624023},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.2466573715209961},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.14757508039474487},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.1152726411819458}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1177/10943420241288242","is_oa":false,"landing_page_url":"https://doi.org/10.1177/10943420241288242","pdf_url":null,"source":{"id":"https://openalex.org/S60606485","display_name":"The International Journal of High Performance Computing Applications","issn_l":"1094-3420","issn":["1094-3420","1741-2846"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320017","host_organization_name":"SAGE Publishing","host_organization_lineage":["https://openalex.org/P4310320017"],"host_organization_lineage_names":["SAGE Publishing"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The International Journal of High Performance Computing Applications","raw_type":"journal-article"},{"id":"pmh:oai:espace2.etsmtl.ca:33597","is_oa":false,"landing_page_url":"https://espace2.etsmtl.ca/id/eprint/33597/","pdf_url":null,"source":{"id":"https://openalex.org/S4306402392","display_name":"Espace \u00c9TS (ETS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1341030882","host_organization_name":"Educational Testing Service","host_organization_lineage":["https://openalex.org/I1341030882"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Non \u00e9valu\u00e9 par les pairs"},{"id":"pmh:oai:osti.gov:2461626","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/2461626","pdf_url":null,"source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:osti.gov:2461626","is_oa":true,"landing_page_url":"https://www.osti.gov/biblio/2461626","pdf_url":null,"source":{"id":"https://openalex.org/S4306402487","display_name":"OSTI OAI (U.S. Department of Energy Office of Scientific and Technical Information)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I139351228","host_organization_name":"Office of Scientific and Technical Information","host_organization_lineage":["https://openalex.org/I139351228"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.5299999713897705}],"awards":[{"id":"https://openalex.org/G3079150831","display_name":null,"funder_award_id":"2004894","funder_id":"https://openalex.org/F4320332169","funder_display_name":"Directorate for Computer and Information Science and Engineering"},{"id":"https://openalex.org/G7358963688","display_name":null,"funder_award_id":"1550588","funder_id":"https://openalex.org/F4320332169","funder_display_name":"Directorate for Computer and Information Science and Engineering"},{"id":"https://openalex.org/G8052219770","display_name":null,"funder_award_id":"17-SC-20-SC","funder_id":"https://openalex.org/F4320337506","funder_display_name":"Advanced Scientific Computing Research"}],"funders":[{"id":"https://openalex.org/F4320332169","display_name":"Directorate for Computer and Information Science and Engineering","ror":"https://ror.org/025kzpk63"},{"id":"https://openalex.org/F4320337506","display_name":"Advanced Scientific Computing Research","ror":"https://ror.org/0012c7r22"},{"id":"https://openalex.org/F4320338284","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W1752227368","https://openalex.org/W1974752004","https://openalex.org/W2013795311","https://openalex.org/W2121400889","https://openalex.org/W2145508367","https://openalex.org/W2529996553","https://openalex.org/W2783428669","https://openalex.org/W2906137127","https://openalex.org/W2913525628","https://openalex.org/W2918778696","https://openalex.org/W2943761765","https://openalex.org/W2950022236","https://openalex.org/W2953302413","https://openalex.org/W2963346670","https://openalex.org/W2966250553","https://openalex.org/W2997905860","https://openalex.org/W2999555012","https://openalex.org/W3006005697","https://openalex.org/W3021718926","https://openalex.org/W3032530299","https://openalex.org/W3046732309","https://openalex.org/W3102176376","https://openalex.org/W3103996135","https://openalex.org/W3104388511","https://openalex.org/W3154968648","https://openalex.org/W3155545561","https://openalex.org/W3164974885","https://openalex.org/W3165027330","https://openalex.org/W3169716534","https://openalex.org/W3198359973","https://openalex.org/W3203287979","https://openalex.org/W3203718464","https://openalex.org/W3204314776","https://openalex.org/W3209285784","https://openalex.org/W4200232000","https://openalex.org/W4200537438","https://openalex.org/W4206443485","https://openalex.org/W4226013321","https://openalex.org/W4237258601","https://openalex.org/W4238826819","https://openalex.org/W4242248730","https://openalex.org/W4280589740","https://openalex.org/W4283268783","https://openalex.org/W4289941695","https://openalex.org/W4300166936","https://openalex.org/W4327550249","https://openalex.org/W4385585393","https://openalex.org/W4387005442","https://openalex.org/W4387527744","https://openalex.org/W4387994442","https://openalex.org/W4388105673","https://openalex.org/W4388581010","https://openalex.org/W4388581250","https://openalex.org/W4391821797"],"related_works":["https://openalex.org/W4318068934","https://openalex.org/W2290952066","https://openalex.org/W2997651281","https://openalex.org/W4320480730","https://openalex.org/W2056528600","https://openalex.org/W4297752063","https://openalex.org/W3128098196","https://openalex.org/W2572977084","https://openalex.org/W4285081369","https://openalex.org/W1989530240"],"abstract_inverted_index":{"Computational":[0],"workflows":[1,18,97],"are":[2],"a":[3,37,49,70,141],"common":[4],"class":[5],"of":[6,17,25,36,72,82,124,143,165,182],"application":[7,61,144],"on":[8,91],"supercomputers,":[9],"yet":[10],"the":[11,33,80,84,95],"loosely":[12],"coupled":[13,139],"and":[14,47,94,127,159],"heterogeneous":[15],"nature":[16],"often":[19],"fails":[20],"to":[21,31,44,57,64],"take":[22],"full":[23],"advantage":[24],"their":[26,60],"capabilities.":[27],"We":[28],"created":[29],"Colmena":[30,54,171],"leverage":[32],"massive":[34],"parallelism":[35],"supercomputer":[38],"by":[39],"using":[40,162],"Artificial":[41],"Intelligence":[42],"(AI)":[43],"learn":[45],"from":[46],"adapt":[48],"workflow":[50,129],"as":[51,69],"it":[52],"executes.":[53],"allows":[55],"scientists":[56],"define":[58],"how":[59],"should":[62],"respond":[63],"events":[65],"(e.g.,":[66],"task":[67],"completion)":[68],"series":[71],"cooperative":[73],"agents.":[74],"In":[75],"this":[76],"paper,":[77],"we":[78,86,98,107],"describe":[79],"design":[81],"Colmena,":[83],"challenges":[85,106],"overcame":[87],"while":[88],"deploying":[89],"applications":[90],"exascale":[92],"systems,":[93],"science":[96,154,161],"have":[99,152],"enhanced":[100],"through":[101,147],"interweaving":[102],"AI.":[103,166],"The":[104],"scaling":[105],"discuss":[108],"include":[109],"developing":[110],"steering":[111,150],"strategies":[112],"that":[113,120,131,170,176],"maximize":[114],"node":[115],"utilization,":[116],"introducing":[117],"data":[118],"fabrics":[119],"reduce":[121],"communication":[122],"overhead":[123],"data-intensive":[125],"tasks,":[126],"implementing":[128],"tasks":[130],"cache":[132],"costly":[133],"operations":[134],"between":[135],"invocations.":[136],"These":[137],"innovations":[138],"with":[140],"variety":[142],"patterns":[145],"accessible":[146],"our":[148],"agent-based":[149],"model":[151],"enabled":[153],"advances":[155],"in":[156],"chemistry,":[157],"biophysics,":[158],"materials":[160],"different":[163],"types":[164],"Our":[167],"vision":[168],"is":[169],"will":[172],"spur":[173],"creative":[174],"solutions":[175],"harness":[177],"AI":[178],"across":[179],"many":[180],"domains":[181],"scientific":[183],"computing.":[184]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3}],"updated_date":"2026-04-20T07:46:08.049788","created_date":"2025-10-10T00:00:00"}