{"id":"https://openalex.org/W2734404947","doi":"https://doi.org/10.1145/3067695.3082533","title":"On the difficulty of benchmarking inductive program synthesis methods","display_name":"On the difficulty of benchmarking inductive program synthesis methods","publication_year":2017,"publication_date":"2017-07-11","ids":{"openalex":"https://openalex.org/W2734404947","doi":"https://doi.org/10.1145/3067695.3082533","mag":"2734404947"},"language":"en","primary_location":{"id":"doi:10.1145/3067695.3082533","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3067695.3082533","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3082533&type=pdf","source":{"id":"https://openalex.org/S4363608771","display_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=3082533&type=pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044538049","display_name":"Edward Pantridge","orcid":"https://orcid.org/0000-0003-0535-5268"},"institutions":[{"id":"https://openalex.org/I4210088658","display_name":"MassMutual Financial Group (United States)","ror":"https://ror.org/0058p7h33","country_code":"US","type":"company","lineage":["https://openalex.org/I4210088658"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Edward Pantridge","raw_affiliation_strings":["MassMutual Financial Group"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MassMutual Financial Group","institution_ids":["https://openalex.org/I4210088658"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083108240","display_name":"Thomas Helmuth","orcid":"https://orcid.org/0000-0002-2330-6809"},"institutions":[{"id":"https://openalex.org/I184889055","display_name":"Washington and Lee University","ror":"https://ror.org/05r9xgf14","country_code":"US","type":"education","lineage":["https://openalex.org/I184889055"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Thomas Helmuth","raw_affiliation_strings":["Washington and Lee University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Washington and Lee University","institution_ids":["https://openalex.org/I184889055"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007077416","display_name":"Nicholas Freitag McPhee","orcid":null},"institutions":[{"id":"https://openalex.org/I2800403580","display_name":"University of Minnesota System","ror":"https://ror.org/03grvy078","country_code":"US","type":"education","lineage":["https://openalex.org/I2800403580"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas Freitag McPhee","raw_affiliation_strings":["University of Minnesota"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Minnesota","institution_ids":["https://openalex.org/I2800403580"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5044142640","display_name":"Lee Spector","orcid":"https://orcid.org/0000-0001-5299-4797"},"institutions":[{"id":"https://openalex.org/I47631775","display_name":"Hampshire College","ror":"https://ror.org/03pe1mm80","country_code":"US","type":"education","lineage":["https://openalex.org/I47631775"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lee Spector","raw_affiliation_strings":["Hampshire College"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hampshire College","institution_ids":["https://openalex.org/I47631775"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2459,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.84452317,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1589","last_page":"1596"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10904","display_name":"Embedded Systems Design Techniques","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8324282169342041},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6265424489974976}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8324282169342041},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6265424489974976},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3067695.3082533","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3067695.3082533","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3082533&type=pdf","source":{"id":"https://openalex.org/S4363608771","display_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3067695.3082533","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3067695.3082533","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3082533&type=pdf","source":{"id":"https://openalex.org/S4363608771","display_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5899999737739563,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G3576524178","display_name":"RI: Small: RUI: Synthesis of Robust Artificial Systems by Adaptive Genetic Programming","funder_award_id":"1617087","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3977699885","display_name":"BSF:2012144:Human-Competitive Evolutionary Computation","funder_award_id":"1331283","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5973345182","display_name":null,"funder_award_id":"1617087,1129139,1331283","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7422149563","display_name":null,"funder_award_id":"1129139 and 1331283","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8045835890","display_name":"UBM-Institutional-Collaborative Research:  Four College Biomath Consortium","funder_award_id":"1129139","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2734404947.pdf","grobid_xml":"https://content.openalex.org/works/W2734404947.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W126267692","https://openalex.org/W1484196467","https://openalex.org/W1506224054","https://openalex.org/W1557360661","https://openalex.org/W1576818901","https://openalex.org/W1792831685","https://openalex.org/W1965446936","https://openalex.org/W1978661986","https://openalex.org/W1980900887","https://openalex.org/W2049311030","https://openalex.org/W2072473332","https://openalex.org/W2132525863","https://openalex.org/W2136414484","https://openalex.org/W2142126234","https://openalex.org/W2146105230","https://openalex.org/W2164611950","https://openalex.org/W2181978274","https://openalex.org/W2511174833","https://openalex.org/W2561490710","https://openalex.org/W2604556510","https://openalex.org/W3004239592"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699","https://openalex.org/W1993948687","https://openalex.org/W2011676020"],"abstract_inverted_index":{"A":[0,171],"variety":[1],"of":[2,16,53,61,91,125,159,178,192],"inductive":[3],"program":[4,29,102,193],"synthesis":[5,30],"(IPS)":[6],"techniques":[7,21],"have":[8,22],"recently":[9],"been":[10,24],"developed,":[11],"emerging":[12],"from":[13],"different":[14,185],"areas":[15],"computer":[17],"science.":[18],"However,":[19],"these":[20,180],"not":[23],"adequately":[25],"compared":[26,115],"on":[27,39,88,95],"general":[28],"problems.":[31],"In":[32],"this":[33],"paper":[34],"we":[35],"compare":[36],"several":[37],"methods":[38,132,181],"problems":[40],"requiring":[41],"solution":[42],"programs":[43],"to":[44,147,183],"handle":[45],"various":[46],"data":[47],"types,":[48],"control":[49],"structures,":[50],"and":[51,110,122,136,151],"numbers":[52],"outputs.":[54],"The":[55,83,114,128,141],"problem":[56],"set":[57],"also":[58],"spans":[59],"levels":[60],"abstraction;":[62],"some":[63],"would":[64,76],"ordinarily":[65,77],"be":[66,78],"approached":[67,79],"using":[68,80],"machine":[69],"code":[70],"or":[71],"assembly":[72],"language,":[73],"while":[74],"others":[75],"high-level":[81],"languages.":[82],"presented":[84],"comparisons":[85],"are":[86,117,156],"focused":[87],"the":[89,97,163,176,189],"possibility":[90],"success;":[92],"that":[93,103,144],"is,":[94],"whether":[96],"system":[98],"can":[99],"produce":[100],"a":[101,167],"passes":[104],"all":[105,108],"tests,":[106],"for":[107],"training":[109],"unseen":[111],"testing":[112],"inputs.":[113],"systems":[116],"Flash":[118],"Fill,":[119],"MagicHaskeller,":[120],"TerpreT,":[121],"two":[123,129],"forms":[124],"genetic":[126,130],"programming.":[127],"programming":[131],"chosen":[133],"were":[134],"PushGP":[135,145],"Grammar":[137,152],"Guided":[138,153],"Genetic":[139,154],"Programming.":[140],"results":[142],"suggest":[143],"and,":[146],"an":[148],"extent,":[149],"TerpreT":[150],"Programming":[155],"more":[157,172],"capable":[158],"finding":[160],"solutions":[161],"than":[162],"others,":[164],"albeit":[165],"at":[166],"higher":[168],"computational":[169],"cost.":[170],"salient":[173],"observation":[174],"is":[175],"difficulty":[177],"comparing":[179],"due":[182],"drastically":[184],"intended":[186],"applications,":[187],"despite":[188],"common":[190],"goal":[191],"synthesis.":[194]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}