{"id":"https://openalex.org/W7137887592","doi":"https://doi.org/10.48550/arxiv.2603.13651","title":"Benchmarking Large Language Models on Reference Extraction and Parsing in the Social Sciences and Humanities","display_name":"Benchmarking Large Language Models on Reference Extraction and Parsing in the Social Sciences and Humanities","publication_year":2026,"publication_date":"2026-03-13","ids":{"openalex":"https://openalex.org/W7137887592","doi":"https://doi.org/10.48550/arxiv.2603.13651"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.13651","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13651","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.13651","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102776532","display_name":"Yurui Zhu","orcid":"https://orcid.org/0000-0001-8753-6606"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Yurui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005227218","display_name":"Giovanni Colavizza","orcid":"https://orcid.org/0000-0002-9806-084X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Colavizza, Giovanni","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5129646836","display_name":"Matteo Romanello","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Romanello, Matteo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.15649999678134918,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.15649999678134918,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12377","display_name":"Digital Humanities and Scholarship","score":0.12229999899864197,"subfield":{"id":"https://openalex.org/subfields/1208","display_name":"Literature and Literary Theory"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0934000015258789,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/parsing","display_name":"Parsing","score":0.8230000138282776},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6078000068664551},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5637000203132629},{"id":"https://openalex.org/keywords/citation","display_name":"Citation","score":0.5519000291824341},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.4925000071525574},{"id":"https://openalex.org/keywords/computational-linguistics","display_name":"Computational linguistics","score":0.3874000012874603},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.38359999656677246},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.3797000050544739}],"concepts":[{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.8230000138282776},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.757099986076355},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6078000068664551},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5996999740600586},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5637000203132629},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.5519000291824341},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5321999788284302},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.4925000071525574},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4278999865055084},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.391400009393692},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.3874000012874603},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.38359999656677246},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.3797000050544739},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.34869998693466187},{"id":"https://openalex.org/C60690694","wikidata":"https://www.wikidata.org/wiki/Q894902","display_name":"Bottom-up parsing","level":4,"score":0.3052000105381012},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2973000109195709},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2897000014781952},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.28529998660087585},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.2653999924659729},{"id":"https://openalex.org/C107763842","wikidata":"https://www.wikidata.org/wiki/Q181040","display_name":"Clef","level":3,"score":0.26409998536109924},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.2572999894618988},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.25279998779296875}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.13651","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13651","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.13651","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13651","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.402229368686676,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Bibliographic":[0],"reference":[1,98,100],"extraction":[2,134],"and":[3,11,26,32,44,77,80,88,102,122,128,144,174,198],"parsing":[4,105,143,146],"are":[5,37],"foundational":[6],"for":[7,184,191],"citation":[8],"indexing,":[9],"linking,":[10],"downstream":[12],"scholarly":[13],"knowledge-graph":[14],"construction.":[15],"However,":[16],"most":[17],"established":[18],"evaluations":[19],"focus":[20],"on":[21,170],"clean,":[22],"English,":[23],"end-of-document":[24],"bibliographies,":[25],"therefore":[27],"underrepresent":[28],"the":[29,148],"Social":[30],"Sciences":[31],"Humanities":[33],"(SSH),":[34],"where":[35],"citations":[36],"frequently":[38],"multilingual,":[39],"embedded":[40],"in":[41],"footnotes,":[42],"abbreviated,":[43],"shaped":[45],"by":[46],"heterogeneous":[47],"historical":[48],"conventions.":[49],"We":[50,90,158],"present":[51],"a":[52,108,116,138],"unified":[53],"benchmark":[54],"that":[55,111,161,175],"targets":[56],"these":[57],"SSH-realistic":[58],"conditions":[59],"across":[60],"three":[61,92],"complementary":[62],"datasets:":[63],"CEX":[64],"(English":[65],"journal":[66],"articles":[67],"spanning":[68],"multiple":[69],"disciplines),":[70],"EXCITE":[71],"(German/English":[72],"documents":[73,200],"with":[74,84],"end-section,":[75],"footnote-only,":[76],"mixed":[78],"regimes),":[79],"LinkedBooks":[81],"(humanities":[82],"references":[83],"strong":[85,117],"stylistic":[86],"variation":[87],"multilinguality).":[89],"evaluate":[91],"tasks":[93],"of":[94],"increasing":[95],"difficulty":[96],"--":[97,106,168,173],"extraction,":[99],"parsing,":[101],"end-to-end":[103,145],"document":[104],"under":[107,155],"schema-constrained":[109],"setup":[110],"enables":[112],"direct":[113],"comparison":[114],"between":[115],"supervised":[118],"pipeline":[119],"baseline":[120],"(GROBID)":[121],"contemporary":[123],"LLMs":[124],"(DeepSeek-V3.1,":[125],"Mistral-Small-3.2-24B,":[126],"Gemma-3-27B-it,":[127],"Qwen3-VL":[129],"(4B-32B":[130],"variants)).":[131],"Across":[132],"datasets,":[133],"largely":[135],"saturates":[136],"beyond":[137],"moderate":[139],"capability":[140],"threshold,":[141],"while":[142,195],"remain":[147],"primary":[149],"bottlenecks":[150],"due":[151],"to":[152,201],"structured-output":[153],"brittleness":[154],"noisy":[156],"layouts.":[157],"further":[159],"show":[160],"lightweight":[162],"LoRA":[163],"adaptation":[164],"yields":[165],"consistent":[166],"gains":[167],"especially":[169],"SSH-heavy":[171],"benchmarks":[172],"segmentation/pipelining":[176],"can":[177],"substantially":[178],"improve":[179],"robustness.":[180],"Finally,":[181],"we":[182],"argue":[183],"hybrid":[185],"deployment":[186],"via":[187],"routing:":[188],"leveraging":[189],"GROBID":[190],"well-structured,":[192],"in-distribution":[193],"PDFs":[194],"escalating":[196],"multilingual":[197],"footnote-heavy":[199],"task-adapted":[202],"LLMs.":[203]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-18T00:00:00"}