{"id":"https://openalex.org/W7166692053","doi":"https://doi.org/10.48550/arxiv.2606.29630","title":"SFBench: The SciFy Scientific Feasibility Benchmark","display_name":"SFBench: The SciFy Scientific Feasibility Benchmark","publication_year":2026,"publication_date":"2026-06-28","ids":{"openalex":"https://openalex.org/W7166692053","doi":"https://doi.org/10.48550/arxiv.2606.29630"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.29630","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29630","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.29630","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139635684","display_name":"Cash Costello","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Costello, Cash","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036270568","display_name":"James Mayfield","orcid":"https://orcid.org/0000-0003-3866-3013"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mayfield, James","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139675102","display_name":"Elsbeth Turcan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Turcan, Elsbeth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139646630","display_name":"Christine Piatko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Piatko, Christine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139713437","display_name":"Christina K. Pikas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pikas, Christina K.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116016312","display_name":"Justin Rokisky","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rokisky, Justin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139674456","display_name":"Sam Scheck","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Scheck, Sam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114635455","display_name":"Chris Ribaudo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ribaudo, Chris","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042142988","display_name":"Ritwik Bose","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bose, Ritwik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5029271331","display_name":"Alex Memory","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Memory, Alex","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.6288999915122986,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.6288999915122986,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.09950000047683716,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.08349999785423279,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8568000197410583},{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.771399974822998},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.6679999828338623},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.620199978351593},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.42480000853538513},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.3804999887943268},{"id":"https://openalex.org/keywords/scientific-reasoning","display_name":"Scientific reasoning","score":0.37139999866485596}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8568000197410583},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.771399974822998},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.6679999828338623},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6502000093460083},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.620199978351593},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4747999906539917},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4675999879837036},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44110000133514404},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.42480000853538513},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.3804999887943268},{"id":"https://openalex.org/C2992562121","wikidata":"https://www.wikidata.org/wiki/Q3817808","display_name":"Scientific reasoning","level":2,"score":0.37139999866485596},{"id":"https://openalex.org/C2992860105","wikidata":"https://www.wikidata.org/wiki/Q7631392","display_name":"Subject matter","level":3,"score":0.3693999946117401},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.36329999566078186},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.3343000113964081},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.32910001277923584},{"id":"https://openalex.org/C2777855551","wikidata":"https://www.wikidata.org/wiki/Q12310021","display_name":"Subject (documents)","level":2,"score":0.30390000343322754},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.30309998989105225},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.29030001163482666},{"id":"https://openalex.org/C2984917352","wikidata":"https://www.wikidata.org/wiki/Q12772819","display_name":"Scientific discovery","level":2,"score":0.28780001401901245},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.27379998564720154},{"id":"https://openalex.org/C112758219","wikidata":"https://www.wikidata.org/wiki/Q16038819","display_name":"Duration (music)","level":2,"score":0.2694999873638153}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.29630","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29630","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.29630","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29630","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,132],"present":[1],"SFBench,":[2],"a":[3,26,31,54],"benchmark":[4,135],"dataset":[5],"for":[6],"evaluating":[7],"systems":[8],"that":[9,39,57,85,111],"assess":[10],"the":[11,83,134],"feasibility":[12,28],"of":[13,38,62],"scientific":[14,64,74],"claims.":[15],"SFBench":[16,127],"includes":[17],"197":[18],"claims":[19,61,68,92],"in":[20,47],"materials":[21],"science,":[22],"each":[23],"annotated":[24],"with":[25,35],"ground-truth":[27],"score":[29],"on":[30,89],"five-point":[32],"scale":[33],"along":[34],"an":[36],"explanation":[37],"assessment.":[40],"The":[41],"collection":[42],"differs":[43],"from":[44,72],"previous":[45],"collections":[46],"several":[48],"important":[49],"ways:":[50],"1)":[51],"it":[52],"defines":[53],"complex":[55],"task":[56],"requires":[58],"reasoning":[59],"over":[60],"varying":[63],"feasibility;":[65],"2)":[66],"its":[67],"are":[69,77,96,129],"not":[70,102],"extracted":[71],"existing":[73],"publications":[75],"but":[76],"created":[78],"de":[79],"novo,":[80],"greatly":[81],"reducing":[82],"chances":[84],"LLMs":[86],"have":[87],"trained":[88],"them;":[90],"3)":[91],"and":[93,106,140,143],"ground":[94],"truth":[95],"established":[97],"by":[98,103],"subject":[99],"matter":[100],"experts,":[101],"artificial":[104],"intelligence;":[105],"4)":[107],"unlike":[108],"many":[109],"benchmarks":[110],"ask":[112,121],"about":[113],"question/answer":[114],"pairs,":[115],"provide":[116],"multiple":[117],"choice":[118],"answers,":[119,126],"or":[120],"questions":[122],"requiring":[123],"short,":[124],"fixed":[125],"explanations":[128],"completely":[130],"open-ended.":[131],"describe":[133],"design,":[136],"data":[137],"creation":[138],"process,":[139],"evaluation":[141],"metrics,":[142],"we":[144],"report":[145],"baseline":[146],"results":[147],"using":[148],"recent":[149],"GPT":[150],"models.":[151]},"counts_by_year":[],"updated_date":"2026-07-01T06:29:00.853634","created_date":"2026-07-01T00:00:00"}