{"id":"https://openalex.org/W4281922005","doi":"https://doi.org/10.48550/arxiv.2205.13902","title":"Sample-Efficient Optimisation with Probabilistic Transformer Surrogates","display_name":"Sample-Efficient Optimisation with Probabilistic Transformer Surrogates","publication_year":2022,"publication_date":"2022-05-27","ids":{"openalex":"https://openalex.org/W4281922005","doi":"https://doi.org/10.48550/arxiv.2205.13902"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2205.13902","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.13902","pdf_url":"https://arxiv.org/pdf/2205.13902","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2205.13902","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018539132","display_name":"Alexandre Maraval","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maraval, Alexandre","raw_affiliation_strings":["Huawei R&D London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei R&D London","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028712584","display_name":"Matthieu Zimmer","orcid":"https://orcid.org/0000-0002-8029-308X","observed_orcids":["https://orcid.org/0000-0002-8029-308X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zimmer, Matthieu","raw_affiliation_strings":["Huawei R&D London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei R&D London","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012157944","display_name":"Antoine Grosnit","orcid":"https://orcid.org/0000-0001-8433-1664","observed_orcids":["https://orcid.org/0000-0001-8433-1664"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Grosnit, Antoine","raw_affiliation_strings":["Huawei R&D London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei R&D London","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000651751","display_name":"Rasul Tutunov","orcid":"https://orcid.org/0000-0002-6495-6437","observed_orcids":["https://orcid.org/0000-0002-6495-6437"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tutunov, Rasul","raw_affiliation_strings":["Huawei R&D London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei R&D London","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Jun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Wang, Jun","raw_affiliation_strings":["University College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034334762","display_name":"Haitham Bou Ammar","orcid":"https://orcid.org/0000-0002-6083-6171","observed_orcids":["https://orcid.org/0000-0002-6083-6171","https://orcid.org/0000-0001-9722-4585"]},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ammar, Haitham Bou","raw_affiliation_strings":["Huawei R&D London University College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Huawei R&D London University College London","institution_ids":["https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9326000213623047,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9326000213623047,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.02419999986886978,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.022099999710917473,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bayesian-optimization","display_name":"Bayesian optimization","score":0.9309999942779541},{"id":"https://openalex.org/keywords/surrogate-modeling","display_name":"surrogate models","score":0.828000009059906},{"id":"https://openalex.org/keywords/black-box-optimization","display_name":"black-box optimization","score":0.8970000147819519},{"id":"https://openalex.org/keywords/sample-efficiency","display_name":"sample efficiency","score":0.6079999804496765},{"id":"https://openalex.org/keywords/exploration-exploitation-trade-off","display_name":"exploration-exploitation balance","score":0.8830000162124634},{"id":"https://openalex.org/keywords/pretraining","display_name":"pre-training","score":0.41600000858306885},{"id":"https://openalex.org/keywords/gaussian-process-priors","display_name":"Gaussian process priors","score":0.5080000162124634},{"id":"https://openalex.org/keywords/predictive-performance","display_name":"predictive performance","score":0.3930000066757202}],"concepts":[{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6537035703659058},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6429075002670288},{"id":"https://openalex.org/C177769412","wikidata":"https://www.wikidata.org/wiki/Q278090","display_name":"Prior probability","level":3,"score":0.4651900827884674},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.4547639489173889},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.44150510430336},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.42190253734588623},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.398661345243454},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3984821140766144},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3900523781776428},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3497435450553894},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.24059945344924927},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2205.13902","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.13902","pdf_url":"https://arxiv.org/pdf/2205.13902","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2205.13902","doi":"https://doi.org/10.48550/arxiv.2205.13902","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2205.13902","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2205.13902","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2205.13902","pdf_url":"https://arxiv.org/pdf/2205.13902","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4281922005.pdf","grobid_xml":"https://content.openalex.org/works/W4281922005.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W1904365287","https://openalex.org/W2055825262","https://openalex.org/W2099201756","https://openalex.org/W2131241448","https://openalex.org/W2132221774","https://openalex.org/W2192203593","https://openalex.org/W2529996553","https://openalex.org/W2904530326","https://openalex.org/W2948210138","https://openalex.org/W2951266961","https://openalex.org/W2965373594","https://openalex.org/W2981851019","https://openalex.org/W2985931096","https://openalex.org/W2998788451","https://openalex.org/W3014339631","https://openalex.org/W3018268080","https://openalex.org/W3043426275","https://openalex.org/W3083389730","https://openalex.org/W3105441357","https://openalex.org/W3116489684","https://openalex.org/W3129638015","https://openalex.org/W3147083170","https://openalex.org/W3162587184","https://openalex.org/W3166028936","https://openalex.org/W3176500072","https://openalex.org/W3184343853","https://openalex.org/W3187864661","https://openalex.org/W3200761280","https://openalex.org/W4226412614","https://openalex.org/W4285043533","https://openalex.org/W4293387938","https://openalex.org/W4313531775","https://openalex.org/W7103481651"],"related_works":["https://openalex.org/W4385750006","https://openalex.org/W4387143446","https://openalex.org/W7133213406","https://openalex.org/W4416102230","https://openalex.org/W7061738714","https://openalex.org/W4415083768","https://openalex.org/W4389073281","https://openalex.org/W4414855950","https://openalex.org/W7162557508","https://openalex.org/W2914196460"],"abstract_inverted_index":{"Faced":[0],"with":[1],"problems":[2],"of":[3,35,128,144,190,241],"increasing":[4],"complexity,":[5],"recent":[6],"research":[7],"in":[8,40,63,137,141,229],"Bayesian":[9],"Optimisation":[10],"(BO)":[11],"has":[12],"focused":[13],"on":[14,74,82,202,212],"adapting":[15],"deep":[16],"probabilistic":[17,38],"models":[18,71],"as":[19,61],"flexible":[20],"alternatives":[21],"to":[22,95,177,217],"Gaussian":[23],"Processes":[24],"(GPs).":[25],"In":[26,186],"a":[27,86,135,157,167,187],"similar":[28],"vein,":[29],"this":[30],"paper":[31],"investigates":[32],"the":[33,116,125,129,145,195],"feasibility":[34],"employing":[36],"state-of-the-art":[37],"transformers":[39],"BO.":[41,219],"Upon":[42],"further":[43],"investigation,":[44],"we":[45,67,99,133,152,192,237],"observe":[46,134],"two":[47,154],"drawbacks":[48],"stemming":[49],"from":[50,89,205],"their":[51,58],"training":[52,102,159],"procedure":[53],"and":[54,165,174,227,246],"loss":[55,130],"definition,":[56],"hindering":[57],"direct":[59],"deployment":[60],"proxies":[62],"black-box":[64],"optimisation.":[65],"First,":[66],"notice":[68],"that":[69,101,198],"these":[70,150],"are":[72],"trained":[73],"uniformly":[75],"distributed":[76,163],"inputs,":[77],"which":[78,119],"impairs":[79],"predictive":[80,138,184],"accuracy":[81,173],"non-uniform":[83],"data":[84,203],"-":[85],"setting":[87],"arising":[88],"any":[90,233],"typical":[91],"BO":[92],"loop":[93],"due":[94],"exploration-exploitation":[96],"trade-offs.":[97],"Second,":[98],"realise":[100],"losses":[103],"(e.g.,":[104],"cross-entropy)":[105],"only":[106,224],"asymptotically":[107],"guarantee":[108],"accurate":[109],"posterior":[110,170],"approximations,":[111],"i.e.,":[112],"after":[113],"arriving":[114],"at":[115],"global":[117],"optimum,":[118],"generally":[120],"cannot":[121],"be":[122],"ensured.":[123],"At":[124],"stationary":[126,180],"points":[127,181],"function,":[131],"however,":[132],"degradation":[136],"performance":[139],"especially":[140],"exploratory":[142],"regions":[143],"input":[146,175],"space.":[147],"To":[148],"tackle":[149],"shortcomings":[151],"introduce":[153],"components:":[155],"1)":[156],"BO-tailored":[158],"prior":[160],"supporting":[161],"non-uniformly":[162],"points,":[164],"2)":[166],"novel":[168],"approximate":[169],"regulariser":[171],"trading-off":[172],"sensitivity":[176],"filter":[178],"favourable":[179],"for":[182,194],"improved":[183],"performance.":[185],"large":[188],"panel":[189],"experiments,":[191],"demonstrate,":[193],"first":[196],"time,":[197],"one":[199],"transformer":[200],"pre-trained":[201,225],"sampled":[204],"random":[206],"GP":[207],"priors":[208],"produces":[209],"competitive":[210],"results":[211],"16":[213],"benchmark":[214],"black-boxes":[215],"compared":[216],"GP-based":[218],"Since":[220],"our":[221],"model":[222],"is":[223],"once":[226],"used":[228],"all":[230],"tasks":[231],"without":[232],"retraining":[234],"and/or":[235],"fine-tuning,":[236],"report":[238],"an":[239],"order":[240],"magnitude":[242],"time-reduction,":[243],"while":[244],"matching":[245],"sometimes":[247],"outperforming":[248],"GPs.":[249]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2022-06-13T00:00:00"}