{"id":"https://openalex.org/W7223216366","doi":"https://doi.org/10.48550/arxiv.2610.28666","title":"Learning Checklists to Reduce Lottery Effects in Scientific Peer Review","display_name":"Learning Checklists to Reduce Lottery Effects in Scientific Peer Review","publication_year":2026,"publication_date":"2026-10-14","ids":{"openalex":"https://openalex.org/W7223216366","doi":"https://doi.org/10.48550/arxiv.2610.28666"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.28666","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.28666","pdf_url":"https://arxiv.org/pdf/2610.28666","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.28666","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005231844","display_name":"Xuanyou Liu","orcid":"https://orcid.org/0009-0007-5326-6194","observed_orcids":["https://orcid.org/0009-0007-5326-6194","https://orcid.org/0009-0001-3207-5432"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liu, Xuanyou","raw_affiliation_strings":["Department of Computer Science , Northwestern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Northwestern University","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5159269824","display_name":"Ziyang Guo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Guo, Ziyang","raw_affiliation_strings":["Department of Computer Science , Northwestern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Northwestern University","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089749790","display_name":"Jason D. Hartline","orcid":"https://orcid.org/0000-0001-5505-6819","observed_orcids":["https://orcid.org/0000-0001-5505-6819"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hartline, Jason","raw_affiliation_strings":["Department of Computer Science , Northwestern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Northwestern University","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068008545","display_name":"Jessica R. Hullman","orcid":"https://orcid.org/0000-0001-6826-3550","observed_orcids":["https://orcid.org/0000-0001-6826-3550"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hullman, Jessica","raw_affiliation_strings":["Department of Computer Science , Northwestern University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Northwestern University","institution_ids":["https://openalex.org/I111979921"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I111979921"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12648","display_name":"Academic Writing and Publishing","score":0.18080000579357147,"subfield":{"id":"https://openalex.org/subfields/3309","display_name":"Library and Information Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12648","display_name":"Academic Writing and Publishing","score":0.18080000579357147,"subfield":{"id":"https://openalex.org/subfields/3309","display_name":"Library and Information Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13516","display_name":"Publishing and Scholarly Communication","score":0.11980000138282776,"subfield":{"id":"https://openalex.org/subfields/3309","display_name":"Library and Information Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.10279999673366547,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/peer-review","display_name":"peer review","score":0.8489999771118164},{"id":"https://openalex.org/keywords/neurips","display_name":"NeurIPS","score":0.6380000114440918},{"id":"https://openalex.org/keywords/iclr","display_name":"ICLR","score":0.8370000123977661}],"concepts":[{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.9329000115394592},{"id":"https://openalex.org/C2777340749","wikidata":"https://www.wikidata.org/wiki/Q6684955","display_name":"Lottery","level":2,"score":0.7846999764442444},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5687000155448914},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4880000054836273},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4553999900817871},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4336000084877014},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4097999930381775},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.37869998812675476},{"id":"https://openalex.org/C138368954","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer review","level":2,"score":0.3756999969482422},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3416000008583069},{"id":"https://openalex.org/C139945424","wikidata":"https://www.wikidata.org/wiki/Q1940696","display_name":"Mean squared error","level":2,"score":0.3240000009536743},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.32100000977516174},{"id":"https://openalex.org/C19619285","wikidata":"https://www.wikidata.org/wiki/Q196372","display_name":"Observational error","level":2,"score":0.2897999882698059},{"id":"https://openalex.org/C40696583","wikidata":"https://www.wikidata.org/wiki/Q989120","display_name":"Type I and type II errors","level":2,"score":0.2614000141620636},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.2538999915122986},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.25380000472068787},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.2533000111579895},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2531999945640564},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.2529999911785126}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.28666","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.28666","pdf_url":"https://arxiv.org/pdf/2610.28666","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.28666","doi":"https://doi.org/10.48550/arxiv.2610.28666","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.28666","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.28666","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.28666","pdf_url":"https://arxiv.org/pdf/2610.28666","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7223216366.pdf","grobid_xml":"https://content.openalex.org/works/W7223216366.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W1553554986","https://openalex.org/W2041457600","https://openalex.org/W2091087160","https://openalex.org/W2091375917","https://openalex.org/W2153691814","https://openalex.org/W2163445888","https://openalex.org/W2924606678","https://openalex.org/W2955229766","https://openalex.org/W3115584932","https://openalex.org/W3201395296","https://openalex.org/W4281392565","https://openalex.org/W4379258778","https://openalex.org/W4379919435","https://openalex.org/W4400734098","https://openalex.org/W4412151072","https://openalex.org/W4414578372","https://openalex.org/W4416035115","https://openalex.org/W4417080452","https://openalex.org/W4417255555","https://openalex.org/W4417474777","https://openalex.org/W7131125571","https://openalex.org/W7131212089","https://openalex.org/W7160402171","https://openalex.org/W7160856271","https://openalex.org/W7164519601","https://openalex.org/W7165359969","https://openalex.org/W7167680454","https://openalex.org/W7203823391"],"related_works":["https://openalex.org/W7220965657","https://openalex.org/W7220669067","https://openalex.org/W7221132137","https://openalex.org/W7126033325","https://openalex.org/W4379258778","https://openalex.org/W4416018011","https://openalex.org/W7172421287","https://openalex.org/W7204451967","https://openalex.org/W7165936077","https://openalex.org/W7162499704"],"abstract_inverted_index":{"Reviewer":[0],"lottery":[1],"error":[2,4,35,112,141],"describes":[3],"due":[5],"to":[6,14,174],"a":[7,21,24,41,47,54,72,115,149,163],"small":[8],"set":[9],"of":[10,17,27,43,56,60,75,114,144,162],"peer":[11],"reviews":[12],"attending":[13],"only":[15],"some":[16],"the":[18,61,68,105,109,131,134,139,155,160,168],"issues":[19],"with":[20,84,102,133,152],"paper":[22,164],"that":[23,70],"larger":[25],"pool":[26],"eligible":[28],"reviewers":[29],"would":[30],"raise.":[31],"We":[32,80],"show":[33],"this":[34,82],"can":[36],"be":[37],"reduced":[38],"by":[39,119],"surfacing":[40],"checklist":[42,107,132,158],"criticisms":[44],"learned":[45,106,156],"from":[46,172],"historical":[48],"review":[49,63,97,101],"corpus.":[50],"Our":[51],"approach":[52,83],"generates":[53],"tree":[55],"checklists":[57],"representing":[58],"coarsenings":[59],"atomic":[62],"claim":[64],"space,":[65],"then":[66],"selects":[67],"coarsening":[69],"maximizes":[71],"theoretically-motivated":[73],"measure":[74],"correlated":[76,136],"agreement":[77,137],"among":[78],"reviewers.":[79],"realize":[81],"an":[85],"LLM-assisted":[86],"pipeline":[87],"and":[88,94,123,127],"test":[89],"it":[90],"on":[91,121,125,128],"NeurIPS":[92,122,157,169],"2021":[93,170],"ICLR":[95],"2022":[96],"data.":[98],"In":[99,148],"simulated":[100],"LLM":[103],"reviewers,":[104],"reduces":[108],"mean":[110],"squared":[111],"(MSE)":[113],"three":[116,145],"reviewer":[117],"committee":[118],"58%":[120],"21%":[124],"ICLR,":[126],"both":[129],"venues":[130],"highest":[135],"has":[138],"lowest":[140],"for":[142],"committees":[143],"or":[146],"more.":[147],"human":[150],"study":[151],"20":[153],"participants,":[154],"lowers":[159],"MSE":[161],"ranking,":[165],"measured":[166],"against":[167],"order,":[171],"0.48":[173],"0.22.":[175]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-23T00:00:00"}