{"id":"https://openalex.org/W4412875478","doi":"https://doi.org/10.1145/3711896.3737419","title":"IdeaBench: Benchmarking Large Language Models for Research Idea Generation","display_name":"IdeaBench: Benchmarking Large Language Models for Research Idea Generation","publication_year":2025,"publication_date":"2025-08-03","ids":{"openalex":"https://openalex.org/W4412875478","doi":"https://doi.org/10.1145/3711896.3737419"},"language":"en","primary_location":{"id":"doi:10.1145/3711896.3737419","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737419","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737419","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737419","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079456168","display_name":"Sikun Guo","orcid":"https://orcid.org/0000-0002-4764-3359"},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sikun Guo","raw_affiliation_strings":["Department of Computer Science, University of Virginia, Charlottesville, VA, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Virginia, Charlottesville, VA, USA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114638568","display_name":"Amir Hassan Shariatmadari","orcid":null},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amir Hassan Shariatmadari","raw_affiliation_strings":["Department of Computer Science, University of Virginia, Charlottesville, VA, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Virginia, Charlottesville, VA, USA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047322230","display_name":"Guangzhi Xiong","orcid":"https://orcid.org/0000-0002-8049-5298"},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Guangzhi Xiong","raw_affiliation_strings":["Department of Computer Science, University of Virginia, Charlottesville, VA, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Virginia, Charlottesville, VA, USA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114638569","display_name":"Albert Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Albert Huang","raw_affiliation_strings":["Department of Computer Science, University of Virginia, Charlottesville, VA, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Virginia, Charlottesville, VA, USA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003207510","display_name":"Myles Kim","orcid":"https://orcid.org/0000-0002-0424-5473"},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Myles Kim","raw_affiliation_strings":["School of Medicine, University of Virginia, Charlottesville, VA, USA"],"affiliations":[{"raw_affiliation_string":"School of Medicine, University of Virginia, Charlottesville, VA, USA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086610701","display_name":"Corey M. Williams","orcid":null},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Corey M. Williams","raw_affiliation_strings":["School of Medicine, University of Virginia, Charlottesville, VA, USA"],"affiliations":[{"raw_affiliation_string":"School of Medicine, University of Virginia, Charlottesville, VA, USA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048872833","display_name":"Stefan Bekiranov","orcid":"https://orcid.org/0000-0002-3177-4346"},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stefan Bekiranov","raw_affiliation_strings":["School of Medicine, University of Virginia, Charlottesville, VA, USA"],"affiliations":[{"raw_affiliation_string":"School of Medicine, University of Virginia, Charlottesville, VA, USA","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013588572","display_name":"Aidong Zhang","orcid":"https://orcid.org/0000-0001-9723-3246"},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aidong Zhang","raw_affiliation_strings":["Department of Computer Science, University of Virginia, Charlottesville, VA, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Virginia, Charlottesville, VA, USA","institution_ids":["https://openalex.org/I51556381"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5079456168"],"corresponding_institution_ids":["https://openalex.org/I51556381"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.09509952,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5888","last_page":"5899"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9796000123023987,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8442388772964478},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7486958503723145},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3805055618286133},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32490023970603943},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.053922832012176514}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8442388772964478},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7486958503723145},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3805055618286133},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32490023970603943},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.053922832012176514},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3711896.3737419","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737419","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737419","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3711896.3737419","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737419","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737419","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4412875478.pdf"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W1593271688","https://openalex.org/W2970785793","https://openalex.org/W2990138404","https://openalex.org/W3099942180","https://openalex.org/W3176456866","https://openalex.org/W4401043863","https://openalex.org/W4402683817","https://openalex.org/W4409982896","https://openalex.org/W6605475740"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"revolutionized":[5],"interactions":[6],"between":[7],"human":[8,104,146,156],"and":[9,23,32,48,65,82,114,144,195],"artificial":[10],"intelligence":[11],"(AI)":[12],"systems,":[13],"demonstrating":[14],"state-of-the-art":[15],"performance":[16],"across":[17,88],"various":[18],"domains,":[19,91],"including":[20],"scientific":[21,206],"discovery":[22],"hypothesis":[24],"generation.":[25],"However,":[26],"the":[27,70,124,129,163],"absence":[28],"of":[29,45,72,165],"a":[30,42,57,62,99,150,190],"comprehensive":[31],"systematic":[33],"evaluation":[34,66],"framework":[35,67],"for":[36,68,193],"LLM-driven":[37],"research":[38,73,90,135,141,200],"idea":[39,74,160],"generation":[40,75],"hinders":[41],"rigorous":[43],"understanding":[44],"their":[46,94],"strengths":[47],"limitations.":[49],"To":[50,137],"address":[51],"this":[52,168],"gap,":[53],"we":[54,121,148,170],"propose":[55,149],"IdeaBench,":[56],"benchmark":[58],"system":[59],"that":[60,102,153,172],"provides":[61],"structured":[63],"dataset":[64,79],"standardizing":[69],"assessment":[71],"by":[76],"LLMs.":[77,166],"Our":[78],"comprises":[80],"titles":[81],"abstracts":[83],"from":[84,128],"2,374":[85],"influential":[86],"papers":[87],"eight":[89],"along":[92],"with":[93,155,162,183],"29,408":[95],"referenced":[96],"works,":[97],"creating":[98],"context-rich":[100],"environment":[101],"mirrors":[103],"researchers'":[105],"ideation":[106,142],"processes.":[107],"By":[108],"profiling":[109],"LLMs":[110,174],"as":[111,189],"domain-specific":[112],"researchers":[113],"grounding":[115],"them":[116],"in":[117,204],"similar":[118],"contextual":[119],"constraints,":[120],"directly":[122],"leverage":[123],"models'":[125],"knowledge":[126],"learned":[127],"pre-training":[130],"stage":[131],"to":[132,158],"generate":[133],"new":[134],"ideas.":[136,186],"systematically":[138],"evaluate":[139],"LLMs'":[140],"capability":[143],"approximate":[145],"assessment,":[147],"reference-based":[151],"metric":[152],"aligns":[154],"judgment":[157],"quantify":[159],"quality":[161],"assistance":[164],"Through":[167],"evaluation,":[169],"find":[171],"while":[173],"excel":[175],"at":[176],"generating":[177,184],"novel":[178],"ideas,":[179],"they":[180],"may":[181],"struggle":[182],"feasible":[185],"IdeaBench":[187],"serves":[188],"critical":[191],"resource":[192],"benchmarking":[194],"comparing":[196],"LLMs,":[197],"ultimately":[198],"advancing":[199],"on":[201],"AI's":[202],"role":[203],"automating":[205],"discovery.":[207]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}