{"id":"https://openalex.org/W7147165704","doi":"https://doi.org/10.48550/arxiv.2603.28769","title":"Spark-LLM-Eval: A Distributed Framework for Statistically Rigorous Large Language Model Evaluation","display_name":"Spark-LLM-Eval: A Distributed Framework for Statistically Rigorous Large Language Model Evaluation","publication_year":2026,"publication_date":"2026-01-18","ids":{"openalex":"https://openalex.org/W7147165704","doi":"https://doi.org/10.48550/arxiv.2603.28769"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.28769","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28769","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.28769","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132562624","display_name":"Subhadip Mitra","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Mitra, Subhadip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5132562624"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.17030000686645508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.17030000686645508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.16509999334812164,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12380","display_name":"Authorship Attribution and Profiling","score":0.1039000004529953,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6804999709129333},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6589000225067139},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.548799991607666},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5029000043869019},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4724000096321106},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.4722999930381775},{"id":"https://openalex.org/keywords/wilcoxon-signed-rank-test","display_name":"Wilcoxon signed-rank test","score":0.36149999499320984},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.3402999937534332}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7505999803543091},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6804999709129333},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6589000225067139},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.548799991607666},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5029000043869019},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.48410001397132874},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4724000096321106},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.4722999930381775},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3971000015735626},{"id":"https://openalex.org/C206041023","wikidata":"https://www.wikidata.org/wiki/Q1751970","display_name":"Wilcoxon signed-rank test","level":3,"score":0.36149999499320984},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.3402999937534332},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.31619998812675476},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.3122999966144562},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30820000171661377},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.2897000014781952},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.2858000099658966},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.27900001406669617},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.27619999647140503},{"id":"https://openalex.org/C176222170","wikidata":"https://www.wikidata.org/wiki/Q5157340","display_name":"Computational statistics","level":2,"score":0.27230000495910645},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.265500009059906},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.26019999384880066},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.25429999828338623}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.28769","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28769","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.28769","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.28769","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Evaluating":[0],"large":[1],"language":[2],"models":[3],"at":[4],"scale":[5,37],"remains":[6],"a":[7,55,69],"practical":[8],"bottleneck":[9],"for":[10,19],"many":[11],"organizations.":[12],"While":[13],"existing":[14],"evaluation":[15,57,67,118,158],"frameworks":[16],"work":[17],"well":[18],"thousands":[20,31],"of":[21,30,34],"examples,":[22],"they":[23],"struggle":[24],"when":[25,40],"datasets":[26],"grow":[27],"to":[28],"hundreds":[29],"or":[32,47,100],"millions":[33],"samples.":[35],"This":[36],"is":[38],"common":[39],"assessing":[41],"model":[42,89],"behavior":[43],"across":[44],"diverse":[45],"domains":[46],"conducting":[48],"comprehensive":[49],"regression":[50],"testing.":[51],"We":[52,136],"present":[53],"Spark-LLM-Eval,":[54],"distributed":[56],"framework":[58,109,155],"built":[59],"natively":[60],"on":[61,104,130],"Apache":[62],"Spark.":[63],"The":[64,108,154],"system":[65,139],"treats":[66],"as":[68,162],"data-parallel":[70],"problem,":[71],"partitioningexamplesacrossexecutorsandaggregatingresultswithproperstatistical":[72],"accounting.":[73],"Beyond":[74],"raw":[75],"throughput,":[76],"we":[77],"emphasize":[78],"statistical":[79,142],"rigor:":[80],"every":[81],"reported":[82],"metric":[83,106,131],"includes":[84],"bootstrap":[85],"confidence":[86],"intervals,":[87],"and":[88,144,156],"comparisons":[90],"come":[91],"with":[92,151],"appropriate":[93],"significance":[94],"tests":[95],"(paired":[96],"t-tests,":[97],"McNemar's":[98],"test,":[99],"Wilcoxon":[101],"signed-rank,":[102],"depending":[103],"the":[105,112,138,141],"type).":[107],"also":[110],"addresses":[111],"cost":[113],"problem":[114],"inherent":[115],"in":[116],"LLM":[117],"through":[119],"content-addressable":[120],"response":[121],"caching":[122],"backed":[123],"by":[124],"Delta":[125],"Lake,":[126],"which":[127],"allows":[128],"iterating":[129],"definitions":[132],"without":[133],"re-running":[134],"inference.":[135],"describe":[137],"architecture,":[140],"methodology,":[143],"report":[145],"benchmark":[146],"results":[147],"showing":[148],"linear":[149],"scaling":[150],"cluster":[152],"size.":[153],"all":[157],"code":[159],"are":[160],"available":[161],"open":[163],"source.":[164]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-02T00:00:00"}