{"id":"https://openalex.org/W7164017023","doi":"https://doi.org/10.48550/arxiv.2606.07936","title":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","display_name":"Illusions of the Gold Standard: A Large-scale Analysis of Human Evaluation Protocols for Long-form Text Generation","publication_year":2026,"publication_date":"2026-06-06","ids":{"openalex":"https://openalex.org/W7164017023","doi":"https://doi.org/10.48550/arxiv.2606.07936"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.07936","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07936","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.07936","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125919280","display_name":"Katelyn Xiaoying Mei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mei, Katelyn Xiaoying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064187871","display_name":"Yi-Li Hsu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hsu, Yi-Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026800427","display_name":"Minjoon Choi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choi, Minjoon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123847142","display_name":"Zongwan Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Zongwan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114482526","display_name":"Chenjun Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Chenjun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138278817","display_name":"Bingbing Wen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wen, Bingbing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001257783","display_name":"Su Lin Blodgett","orcid":"https://orcid.org/0000-0002-9861-3483"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Blodgett, Su Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138223740","display_name":"Lucy Lu Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Lucy Lu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14470000565052032,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14470000565052032,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12377","display_name":"Digital Humanities and Scholarship","score":0.07930000126361847,"subfield":{"id":"https://openalex.org/subfields/1208","display_name":"Literature and Literary Theory"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.07590000331401825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.6553999781608582},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5575000047683716},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5358999967575073},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5084999799728394},{"id":"https://openalex.org/keywords/human-reliability","display_name":"Human reliability","score":0.45989999175071716},{"id":"https://openalex.org/keywords/illusion","display_name":"Illusion","score":0.41339999437332153}],"concepts":[{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.6553999781608582},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6348999738693237},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5575000047683716},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5358999967575073},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5084999799728394},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4616999924182892},{"id":"https://openalex.org/C191147762","wikidata":"https://www.wikidata.org/wiki/Q186289","display_name":"Human reliability","level":3,"score":0.45989999175071716},{"id":"https://openalex.org/C184047640","wikidata":"https://www.wikidata.org/wiki/Q182593","display_name":"Illusion","level":2,"score":0.41339999437332153},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.4043999910354614},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.32440000772476196},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.30300000309944153},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.289900004863739},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.2800999879837036},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2574999928474426},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.2533000111579895},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.07936","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07936","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.07936","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07936","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.6781667470932007,"id":"https://metadata.un.org/sdg/5","display_name":"Gender equality"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Human":[0],"evaluation":[1,46,87,112],"plays":[2],"a":[3,41,60,76],"critical":[4],"role":[5],"in":[6,33,53,148],"assessing":[7],"the":[8,14,101],"quality":[9],"of":[10,18,44,64,78,85,107,110],"generated":[11],"text.":[12],"However,":[13],"reliability":[15],"and":[16,24,67,89,98,122,127,145,154],"reproducibility":[17,84],"these":[19,91,135],"evaluations":[20],"depend":[21],"on":[22,134],"transparent":[23,144],"well-documented":[25],"protocols":[26,47],"--":[27],"details":[28],"that":[29],"are":[30],"frequently":[31],"missing":[32],"current":[34],"practice.":[35],"In":[36],"this":[37],"work,":[38],"we":[39,137],"conduct":[40],"large-scale":[42],"analysis":[43,69,152],"human":[45,86,111],"for":[48,70],"evaluating":[49],"long-form":[50],"generation":[51],"tasks":[52],"*CL":[54],"conference":[55],"publications":[56],"from":[57],"2023--2025,":[58],"including":[59],"full":[61],"manual":[62],"review":[63],"284":[65],"papers":[66],"LLM-assisted":[68],"another":[71],"1.8k+":[72],"papers.":[73],"We":[74,103],"define":[75],"set":[77],"20":[79],"reportable":[80],"criteria":[81,92],"related":[82],"to":[83,93,116,141],"studies,":[88],"apply":[90],"systematically":[94],"examine":[95],"reporting":[96,147],"norms":[97],"practices":[99],"within":[100],"community.":[102],"find":[104],"widespread":[105],"under-reporting":[106],"important":[108],"aspects":[109],"study":[113],"design,":[114],"leading":[115],"ambiguity":[117],"about":[118],"what":[119],"was":[120],"measured":[121],"how,":[123],"who":[124],"contributed":[125],"judgments,":[126],"how":[128],"judgments":[129],"should":[130],"be":[131,158],"interpreted.":[132],"Based":[133],"findings,":[136],"outline":[138],"actionable":[139],"recommendations":[140],"support":[142],"more":[143],"reproducible":[146],"future":[149],"research.":[150],"Our":[151],"code":[153],"annotated":[155],"dataset":[156],"can":[157],"found":[159],"at:":[160],"https://github.com/larchlab/Illusions-of-the-Gold-Standard":[161]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-10T00:00:00"}