{"id":"https://openalex.org/W4388926382","doi":"https://doi.org/10.48550/arxiv.2311.11944","title":"FinanceBench: A New Benchmark for Financial Question Answering","display_name":"FinanceBench: A New Benchmark for Financial Question Answering","publication_year":2023,"publication_date":"2023-11-20","ids":{"openalex":"https://openalex.org/W4388926382","doi":"https://doi.org/10.48550/arxiv.2311.11944"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2311.11944","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.11944","pdf_url":"https://arxiv.org/pdf/2311.11944","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2311.11944","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093321104","display_name":"Pranab Islam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Islam, Pranab","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024319800","display_name":"A. Kannappan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kannappan, Anand","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016956470","display_name":"Douwe Kiela","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kiela, Douwe","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014742037","display_name":"Rebecca Qian","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qian, Rebecca","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023054628","display_name":"Nino Scherrer","orcid":"https://orcid.org/0000-0001-5976-4257","observed_orcids":["https://orcid.org/0000-0001-5976-4257"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Scherrer, Nino","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084936290","display_name":"Bertie Vidgen","orcid":"https://orcid.org/0000-0002-7892-0814","observed_orcids":["https://orcid.org/0000-0002-7892-0814"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vidgen, Bertie","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":406,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9702000021934509,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9702000021934509,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.01549999974668026,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/financial-question-answering","display_name":"financial question answering","score":0.8130000233650208},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7689999938011169},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.6179999709129333},{"id":"https://openalex.org/keywords/listed-companies","display_name":"listed companies","score":0.8880000114440918},{"id":"https://openalex.org/keywords/evidence-retrieval","display_name":"evidence retrieval","score":0.5419999957084656},{"id":"https://openalex.org/keywords/hallucinations","display_name":"hallucinations","score":0.8109999895095825},{"id":"https://openalex.org/keywords/gpt-4-turbo","display_name":"GPT-4 Turbo","score":0.8230000138282776},{"id":"https://openalex.org/keywords/llama-2","display_name":"Llama 2","score":0.7570000290870667},{"id":"https://openalex.org/keywords/claude-2","display_name":"Claude 2","score":0.8510000109672546},{"id":"https://openalex.org/keywords/vector-store","display_name":"vector store","score":0.6830000281333923}],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6518712043762207},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6437101364135742},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.6273888945579529},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6230565905570984},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.51441890001297},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.5018994808197021},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.47350603342056274},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.4250751733779907},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.3737345337867737},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2810963988304138},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2558927536010742},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.22077032923698425},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.1775510013103485},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.14773809909820557},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2311.11944","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.11944","pdf_url":"https://arxiv.org/pdf/2311.11944","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2311.11944","doi":"https://doi.org/10.48550/arxiv.2311.11944","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2311.11944","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2311.11944","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.11944","pdf_url":"https://arxiv.org/pdf/2311.11944","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4388926382.pdf","grobid_xml":"https://content.openalex.org/works/W4388926382.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7115358900","https://openalex.org/W7123837676","https://openalex.org/W4388684812","https://openalex.org/W6910773300","https://openalex.org/W4407358895","https://openalex.org/W4400374373","https://openalex.org/W4407012932","https://openalex.org/W7150893500","https://openalex.org/W4415796224","https://openalex.org/W7223353990"],"abstract_inverted_index":{"FinanceBench":[0,36],"is":[1],"a":[2,42,60,86,119],"first-of-its-kind":[3],"test":[4,65],"suite":[5],"for":[6,112,150,178],"evaluating":[7],"the":[8,69],"performance":[9,62],"of":[10,45,68,88,129],"LLMs":[11,108],"on":[12,85],"open":[13],"book":[14],"financial":[15,113,161],"question":[16],"answering":[17],"(QA).":[18],"It":[19],"comprises":[20],"10,231":[21],"questions":[22,34],"about":[23],"publicly":[24],"traded":[25],"companies,":[26],"with":[27,78,118],"corresponding":[28],"answers":[29,97],"and":[30,40,53,76,81,93,157],"evidence":[31,144],"strings.":[32],"The":[33,99],"in":[35,142],"are":[37,48,101,148],"ecologically":[38],"valid":[39],"cover":[41],"diverse":[43],"set":[44],"scenarios.":[46],"They":[47],"intended":[49],"to":[50,55,57,126,140,154],"be":[51],"clear-cut":[52],"straightforward":[54],"answer":[56,127],"serve":[58],"as":[59,135,172],"minimum":[61],"standard.":[63],"We":[64,104,163],"16":[66],"state":[67],"art":[70],"model":[71],"configurations":[72],"(including":[73],"GPT-4-Turbo,":[74],"Llama2":[75],"Claude2,":[77],"vector":[79],"stores":[80],"long":[82],"context":[83,138],"prompts)":[84],"sample":[87],"150":[89],"cases":[90,100],"from":[91],"FinanceBench,":[92],"manually":[94],"review":[95],"their":[96,176],"(n=2,400).":[98],"available":[102],"open-source.":[103],"show":[105],"that":[106,165,174],"existing":[107],"have":[109],"clear":[110],"limitations":[111],"QA.":[114],"Notably,":[115],"GPT-4-Turbo":[116],"used":[117],"retrieval":[120],"system":[121],"incorrectly":[122],"answered":[123],"or":[124],"refused":[125],"81%":[128],"questions.":[130],"While":[131],"augmentation":[132],"techniques":[133],"such":[134,171],"using":[136],"longer":[137],"window":[139],"feed":[141],"relevant":[143],"improve":[145],"performance,":[146],"they":[147],"unrealistic":[149],"enterprise":[151],"settings":[152],"due":[153],"increased":[155],"latency":[156],"cannot":[158],"support":[159],"larger":[160],"documents.":[162],"find":[164],"all":[166],"models":[167],"examined":[168],"exhibit":[169],"weaknesses,":[170],"hallucinations,":[173],"limit":[175],"suitability":[177],"use":[179],"by":[180],"enterprises.":[181]},"counts_by_year":[{"year":2026,"cited_by_count":252},{"year":2025,"cited_by_count":109},{"year":2024,"cited_by_count":44},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-11-23T00:00:00"}