{"id":"https://openalex.org/W7214791998","doi":"https://doi.org/10.48550/arxiv.2609.33233","title":"Inspire: Benchmarking Scientific Literature Search for Open Research Problems","display_name":"Inspire: Benchmarking Scientific Literature Search for Open Research Problems","publication_year":2026,"publication_date":"2026-09-27","ids":{"openalex":"https://openalex.org/W7214791998","doi":"https://doi.org/10.48550/arxiv.2609.33233"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.33233","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.33233","pdf_url":"https://arxiv.org/pdf/2609.33233","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.33233","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5153460610","display_name":"Jianrong Ding","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Ding, Jianrong","raw_affiliation_strings":["Department of Computer Science and Engineering , The Chinese University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , The Chinese University of Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153482364","display_name":"Zhengyan Shi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shi, Zhengyan","raw_affiliation_strings":["Department of Computer Science and Engineering , The Chinese University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , The Chinese University of Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153525818","display_name":"Jianyuan Zhong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Zhong, Jianyuan","raw_affiliation_strings":["Department of Computer Science and Engineering , The Chinese University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , The Chinese University of Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102742709","display_name":"Kai Qiu","orcid":"https://orcid.org/0000-0003-2071-4425","observed_orcids":["https://orcid.org/0000-0003-2071-4425"]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiu, Kai","raw_affiliation_strings":["Microsoft Research Asia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153516492","display_name":"Qi Dai","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dai, Qi","raw_affiliation_strings":["Microsoft Research Asia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153453310","display_name":"Yifan Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang, Yifan","raw_affiliation_strings":["Microsoft Research Asia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5153530336","display_name":"Chong Luo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Luo, Chong","raw_affiliation_strings":["Microsoft Research Asia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153527786","display_name":"Qiang Xu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xu, Qiang","raw_affiliation_strings":["Department of Computer Science and Engineering , The Chinese University of Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering , The Chinese University of Hong Kong","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.006899999920278788,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scientific-information-retrieval","display_name":"scientific information retrieval","score":0.906000018119812},{"id":"https://openalex.org/keywords/open-problems","display_name":"open problems","score":0.8799999952316284},{"id":"https://openalex.org/keywords/ai-agents","display_name":"AI agents","score":0.5210000276565552},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.6949999928474426},{"id":"https://openalex.org/keywords/search-trajectories","display_name":"search trajectories","score":0.5889999866485596},{"id":"https://openalex.org/keywords/antecedent-selection","display_name":"antecedent selection","score":0.6779999732971191},{"id":"https://openalex.org/keywords/document-ranking","display_name":"document ranking","score":0.5350000262260437},{"id":"https://openalex.org/keywords/computer-science-literature","display_name":"computer science literature","score":0.45100000500679016}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7720999717712402},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.7093999981880188},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6582000255584717},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6301000118255615},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.5907999873161316},{"id":"https://openalex.org/C2781256819","wikidata":"https://www.wikidata.org/wiki/Q16828835","display_name":"Antecedent (behavioral psychology)","level":2,"score":0.503600001335144},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.44850000739097595},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4480000138282776},{"id":"https://openalex.org/C10347200","wikidata":"https://www.wikidata.org/wiki/Q1960297","display_name":"Hindsight bias","level":2,"score":0.38359999656677246},{"id":"https://openalex.org/C2780821815","wikidata":"https://www.wikidata.org/wiki/Q5340806","display_name":"Portfolio","level":2,"score":0.37619999051094055},{"id":"https://openalex.org/C2781083858","wikidata":"https://www.wikidata.org/wiki/Q17327049","display_name":"Scientific literature","level":2,"score":0.3734000027179718},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3474999964237213},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.34049999713897705},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.2978000044822693},{"id":"https://openalex.org/C2778355321","wikidata":"https://www.wikidata.org/wiki/Q17079427","display_name":"Identity (music)","level":2,"score":0.28630000352859497},{"id":"https://openalex.org/C2984917352","wikidata":"https://www.wikidata.org/wiki/Q12772819","display_name":"Scientific discovery","level":2,"score":0.2833000123500824},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.27959999442100525},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2547999918460846}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.33233","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.33233","pdf_url":"https://arxiv.org/pdf/2609.33233","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.33233","doi":"https://doi.org/10.48550/arxiv.2609.33233","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.33233","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.33233","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.33233","pdf_url":"https://arxiv.org/pdf/2609.33233","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214791998.pdf","grobid_xml":"https://content.openalex.org/works/W7214791998.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7216217917","https://openalex.org/W7223417879","https://openalex.org/W7165399758","https://openalex.org/W7204275524","https://openalex.org/W7222969410","https://openalex.org/W7140117608","https://openalex.org/W7204524933","https://openalex.org/W7167039892","https://openalex.org/W7165640783","https://openalex.org/W7215027815"],"abstract_inverted_index":{"Scientific":[0],"literature":[1,32],"search":[2,30,101,138,199],"often":[3],"begins":[4],"with":[5,46,180],"an":[6,73,157,209],"open":[7,74],"research":[8,38,44,68],"problem":[9],"rather":[10],"than":[11,160],"a":[12,17,24,43,47,53,136,162,223],"known":[13],"target":[14,81],"paper":[15,55,82],"or":[16],"fixed":[18],"candidate":[19],"set.":[20],"We":[21,187],"introduce":[22],"INSPIRE,":[23],"benchmark":[25,207],"for":[26],"evaluating":[27],"agents":[28,153],"that":[29,65,151,195,205],"prior":[31,167],"to":[33,103],"make":[34],"progress":[35],"on":[36],"solution-redacted":[37],"problems.":[39],"Each":[40],"instance":[41],"pairs":[42],"brief":[45],"target-specific":[48],"cutoff":[49],"three":[50,105],"months":[51],"before":[52],"later":[54],"and":[56,83,123,140,185,193,219],"evaluates":[57],"ranked":[58],"outputs":[59],"against":[60],"graded":[61],"cited":[62,87],"antecedents":[63,88,112,120],"from":[64,91],"paper's":[66],"realized":[67],"lineage.":[69],"Search":[70],"proceeds":[71],"over":[72],"corpus,":[75],"while":[76],"the":[77,80,92,142,176,206,226],"identity":[78],"of":[79,85,165,234],"membership":[84],"its":[86,230],"remain":[89],"hidden":[90],"agent.":[93],"Beyond":[94],"end-to-end":[95,217],"retrieval":[96],"quality,":[97],"INSPIRE":[98,213],"uses":[99],"logged":[100],"trajectories":[102],"distinguish":[104],"coupled":[106],"stages:":[107],"resource":[108,173],"exposure,":[109],"whether":[110,118],"useful":[111],"are":[113,121,129],"surfaced":[114],"during":[115],"search;":[116],"selection,":[117],"exposed":[119],"retained;":[122],"ranking,":[124],"how":[125],"effectively":[126],"retained":[127],"papers":[128],"ordered.":[130],"Across":[131],"476":[132],"computer-science":[133],"targets":[134],"under":[135],"shared":[137],"interface":[139],"budget,":[141],"strongest":[143],"evaluated":[144],"agent":[145,227],"achieves":[146],"0.284":[147],"nDCG@10.":[148],"Results":[149],"show":[150,194],"current":[152],"more":[154],"readily":[155],"recover":[156],"isolated":[158],"antecedent":[159],"assemble":[161],"broader":[163],"portfolio":[164],"relevant":[166],"work.":[168],"The":[169],"stagewise":[170],"analysis":[171],"identifies":[172],"exposure":[174],"as":[175],"largest":[177],"observed":[178],"bottleneck,":[179],"further":[181],"losses":[182],"in":[183,222],"selection":[184],"ranking.":[186],"additionally":[188],"construct":[189,229],"replay-valid":[190],"hindsight":[191],"demonstrations":[192],"they":[196],"improve":[197],"held-out":[198],"without":[200],"changing":[201],"test-time":[202],"information,":[203],"establishing":[204],"provides":[208],"actionable":[210],"learning":[211],"signal.":[212],"therefore":[214],"enables":[215],"both":[216],"comparison":[218],"stage-resolved":[220],"diagnosis":[221],"setting":[224],"where":[225],"must":[228],"own":[231],"working":[232],"criterion":[233],"relevance.":[235]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-30T00:00:00"}