{"id":"https://openalex.org/W7203713620","doi":"https://doi.org/10.5281/zenodo.21988815","title":"Multi-judge rescoring of small-LLM response aggregation: code and data","display_name":"Multi-judge rescoring of small-LLM response aggregation: code and data","publication_year":2026,"publication_date":"2026-08-18","ids":{"openalex":"https://openalex.org/W7203713620","doi":"https://doi.org/10.5281/zenodo.21988815"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.21988815","doi":"https://doi.org/10.5281/zenodo.21988815","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.21988815","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"volume":null,"type":"dataset","is_traditional":false,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.21988815","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100655391","display_name":"Kihwan Kim","orcid":"https://orcid.org/0000-0002-0502-9545","observed_orcids":["https://orcid.org/0000-0002-0502-9545"]},"institutions":[{"id":"https://openalex.org/I204974687","display_name":"Dong-Eui University","ror":"https://ror.org/059g69b28","country_code":"KR","type":"education","lineage":["https://openalex.org/I204974687"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kim, Ki-Hwan","raw_affiliation_strings":["Dong-Eui University"],"raw_orcid":"https://orcid.org/0000-0002-0502-9545","affiliations":[{"raw_affiliation_string":"Dong-Eui University","institution_ids":["https://openalex.org/I204974687"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5147389544","display_name":"Young-Jin Kang","orcid":"https://orcid.org/0009-0009-8774-7804","observed_orcids":["https://orcid.org/0009-0009-8774-7804"]},"institutions":[{"id":"https://openalex.org/I204974687","display_name":"Dong-Eui University","ror":"https://ror.org/059g69b28","country_code":"KR","type":"education","lineage":["https://openalex.org/I204974687"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Kang, Young-Jin","raw_affiliation_strings":["Dong-Eui University"],"raw_orcid":"https://orcid.org/0009-0009-8774-7804","affiliations":[{"raw_affiliation_string":"Dong-Eui University","institution_ids":["https://openalex.org/I204974687"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038399094","display_name":"Seok Chan Jeong","orcid":"https://orcid.org/0000-0002-8696-4489","observed_orcids":["https://orcid.org/0000-0002-8696-4489"]},"institutions":[{"id":"https://openalex.org/I204974687","display_name":"Dong-Eui University","ror":"https://ror.org/059g69b28","country_code":"KR","type":"education","lineage":["https://openalex.org/I204974687"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jeong, Seok Chan","raw_affiliation_strings":["Dong-Eui University"],"raw_orcid":"https://orcid.org/0000-0002-8696-4489","affiliations":[{"raw_affiliation_string":"Dong-Eui University","institution_ids":["https://openalex.org/I204974687"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I204974687"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.46239998936653137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.46239998936653137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.24709999561309814,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.14069999754428864,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/small-language-models","display_name":"small language models","score":0.7799999713897705},{"id":"https://openalex.org/keywords/response-aggregation","display_name":"response aggregation","score":0.906000018119812},{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.5540000200271606},{"id":"https://openalex.org/keywords/claude","display_name":"Claude","score":0.5839999914169312},{"id":"https://openalex.org/keywords/mistral","display_name":"Mistral","score":0.609000027179718},{"id":"https://openalex.org/keywords/qwen","display_name":"Qwen","score":0.984000027179718},{"id":"https://openalex.org/keywords/gemma","display_name":"Gemma","score":0.7990000247955322},{"id":"https://openalex.org/keywords/inter-judge-agreement","display_name":"inter-judge agreement","score":0.7630000114440918},{"id":"https://openalex.org/keywords/rank-reversal","display_name":"rank reversal","score":0.8500000238418579},{"id":"https://openalex.org/keywords/multilingual-question-answering","display_name":"multilingual question answering","score":0.8090000152587891}],"concepts":[{"id":"https://openalex.org/C206041023","wikidata":"https://www.wikidata.org/wiki/Q1751970","display_name":"Wilcoxon signed-rank test","level":3,"score":0.5785999894142151},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.453000009059906},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.43779999017715454},{"id":"https://openalex.org/C21308566","wikidata":"https://www.wikidata.org/wiki/Q7169365","display_name":"Permutation (music)","level":2,"score":0.39640000462532043},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3898000121116638},{"id":"https://openalex.org/C154504017","wikidata":"https://www.wikidata.org/wiki/Q853614","display_name":"Identifier","level":2,"score":0.38530001044273376},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.3808000087738037},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.365200012922287},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.35659998655319214},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34060001373291016},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3167000114917755},{"id":"https://openalex.org/C47385372","wikidata":"https://www.wikidata.org/wiki/Q7098943","display_name":"Optimal distinctiveness theory","level":2,"score":0.31380000710487366},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.31029999256134033},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.30000001192092896},{"id":"https://openalex.org/C101601086","wikidata":"https://www.wikidata.org/wiki/Q3753228","display_name":"Rank correlation","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C44083865","wikidata":"https://www.wikidata.org/wiki/Q3853443","display_name":"Mean reciprocal rank","level":2,"score":0.2590999901294708},{"id":"https://openalex.org/C176730311","wikidata":"https://www.wikidata.org/wiki/Q867668","display_name":"Multiple choice","level":3,"score":0.25270000100135803}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.5281/zenodo.21988815","doi":"https://doi.org/10.5281/zenodo.21988815","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.21988815","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},{"id":"doi:10.5281/zenodo.21988816","doi":"https://doi.org/10.5281/zenodo.21988816","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.21988816","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},{"id":"doi:10.5281/zenodo.23757812","doi":"https://doi.org/10.5281/zenodo.23757812","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23757812","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},{"id":"doi:10.5281/zenodo.23833492","doi":"https://doi.org/10.5281/zenodo.23833492","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23833492","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"}],"best_oa_location":{"id":"doi:10.5281/zenodo.21988815","doi":"https://doi.org/10.5281/zenodo.21988815","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.21988815","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7108336250","https://openalex.org/W7219967721","https://openalex.org/W7090848549","https://openalex.org/W4412886854","https://openalex.org/W7172383024","https://openalex.org/W7214926339","https://openalex.org/W4415337163","https://openalex.org/W7204183713","https://openalex.org/W7214756950","https://openalex.org/W7221035769"],"abstract_inverted_index":{"Corrected":[0],"release.":[1],"The":[2,84],"Claude":[3],"judge":[4,18,72,112],"scores":[5,28,113],"in":[6],"v1.0.0":[7],"are":[8,74,142],"invalid":[9],"because":[10],"the":[11,117,121,134,149],"scoring":[12],"wrapper":[13],"replaced":[14],"failed":[15,71],"or":[16],"unparseable":[17],"calls":[19,73],"with":[20,63,107],"a":[21,98],"score":[22],"of":[23,145],"3":[24],"without":[25],"recording":[26],"which":[27],"were":[29],"substitutions.":[30],"In":[31],"this":[32],"version":[33],"every":[34,157],"answer":[35],"is":[36,82,155,160],"rescored":[37],"by":[38],"seven":[39,91],"judges":[40],"from":[41,90,133],"four":[42],"vendor":[43],"families":[44],"(Claude":[45],"Haiku":[46],"4.5,":[47],"Sonnet":[48],"4.6,":[49],"Opus":[50],"4.7;":[51],"Gemma":[52],"4":[53],"26B;":[54],"Mistral":[55],"Small":[56],"3.2":[57],"24B;":[58],"Qwen3.5":[59],"9B;":[60],"Qwen2.5-VL":[61],"32B)":[62],"an":[64],"identical":[65],"1":[66],"to":[67,95,140],"5":[68],"rubric,":[69],"and":[70,109,116,126,130],"recorded":[75],"as":[76],"empty":[77],"cells;":[78],"no":[79],"default":[80],"value":[81],"substituted.":[83],"record":[85],"contains":[86],"2,800":[87],"generated":[88],"answers":[89],"response-aggregation":[92],"configurations":[93,144],"(B0":[94],"B6)":[96],"over":[97],"400-query":[99],"multilingual":[100],"test":[101],"set":[102],"(Korean,":[103],"English,":[104],"German,":[105],"Japanese),":[106],"latency":[108],"cost;":[110],"19,600":[111],"(three":[114],"empty);":[115],"scripts":[118],"for":[119],"rescoring,":[120],"generalizability":[122],"study,":[123],"rank-reversal":[124],"counts,":[125],"inter-judge":[127],"agreement.":[128],"B0":[129],"B6":[131],"derive":[132],"authors'":[135,150],"own":[136],"earlier":[137],"systems;":[138],"B1":[139],"B5":[141],"approximate":[143],"published":[146],"methods":[147],"within":[148],"harness.":[151],"Source":[152],"question":[153],"text":[154],"withheld;":[156],"item":[158],"identifier":[159],"published.":[161]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-08-19T00:00:00"}