{"id":"https://openalex.org/W3045321166","doi":"https://doi.org/10.1162/tacl_a_00373","title":"SummEval: Re-evaluating Summarization Evaluation","display_name":"SummEval: Re-evaluating Summarization Evaluation","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3045321166","doi":"https://doi.org/10.1162/tacl_a_00373","mag":"3045321166"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00373","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00373","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00373/1923949/tacl_a_00373.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00373/1923949/tacl_a_00373.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023648764","display_name":"Alexander R. Fabbri","orcid":"https://orcid.org/0000-0002-4670-704X"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander R. Fabbri","raw_affiliation_strings":["Yale University, United States. alexander.fabbri@yale.edu","Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University, United States. alexander.fabbri@yale.edu","institution_ids":["https://openalex.org/I32971472"]},{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070892989","display_name":"Wojciech Kry\u015bci\u0144ski","orcid":null},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]},{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wojciech Kry\u015bci\u0144ski","raw_affiliation_strings":["Salesforce Research, United States. kryscinski@salesforce.com","Yale University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, United States. kryscinski@salesforce.com","institution_ids":["https://openalex.org/I4210155268"]},{"raw_affiliation_string":"Yale University","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008147177","display_name":"Bryan McCann","orcid":"https://orcid.org/0000-0001-9937-1730"},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bryan McCann","raw_affiliation_strings":["Salesforce Research, United States. bryan.mccann.is@gmail.com","Salesforce#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, United States. bryan.mccann.is@gmail.com","institution_ids":["https://openalex.org/I4210155268"]},{"raw_affiliation_string":"Salesforce#TAB#","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032046813","display_name":"Caiming Xiong","orcid":"https://orcid.org/0000-0003-0349-8628"},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Caiming Xiong","raw_affiliation_strings":["Salesforce Research, United States. cxiong@salesforce.com","Salesforce#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, United States. cxiong@salesforce.com","institution_ids":["https://openalex.org/I4210155268"]},{"raw_affiliation_string":"Salesforce#TAB#","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059955534","display_name":"Richard Socher","orcid":"https://orcid.org/0000-0002-3577-639X"},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Richard Socher","raw_affiliation_strings":["Salesforce Research, United States. richard@socher.org","Salesforce#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, United States. richard@socher.org","institution_ids":["https://openalex.org/I4210155268"]},{"raw_affiliation_string":"Salesforce#TAB#","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081787254","display_name":"Dragomir Radev","orcid":"https://orcid.org/0000-0001-7830-6489"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]},{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dragomir Radev","raw_affiliation_strings":["Salesforce Research, United States. dragomir.radev@yale.edu","Yale University, United States","Salesforce#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, United States. dragomir.radev@yale.edu","institution_ids":["https://openalex.org/I4210155268"]},{"raw_affiliation_string":"Yale University, United States","institution_ids":["https://openalex.org/I32971472"]},{"raw_affiliation_string":"Salesforce#TAB#","institution_ids":["https://openalex.org/I4210155268"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.496,"has_fulltext":false,"cited_by_count":47,"citation_normalized_percentile":{"value":0.96402146,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"9","issue":null,"first_page":"391","last_page":"409"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.973908543586731},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8532556295394897},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7186522483825684},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5303070545196533},{"id":"https://openalex.org/keywords/multi-document-summarization","display_name":"Multi-document summarization","score":0.5052958130836487},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.4859255850315094},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40191590785980225},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36805248260498047},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32557666301727295}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.973908543586731},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8532556295394897},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7186522483825684},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5303070545196533},{"id":"https://openalex.org/C134714966","wikidata":"https://www.wikidata.org/wiki/Q6934448","display_name":"Multi-document summarization","level":3,"score":0.5052958130836487},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.4859255850315094},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40191590785980225},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36805248260498047},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32557666301727295},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1162/tacl_a_00373","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00373","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00373/1923949/tacl_a_00373.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2007.12626","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.12626","pdf_url":"https://arxiv.org/pdf/2007.12626","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"mag:3045321166","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2007.12626.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:doaj.org/article:3a815d182a8440e1ac02f25f1d9da002","is_oa":false,"landing_page_url":"https://doaj.org/article/3a815d182a8440e1ac02f25f1d9da002","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 9, Pp 391-409 (2021)","raw_type":"article"},{"id":"doi:10.48550/arxiv.2007.12626","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2007.12626","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00373","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00373","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00373/1923949/tacl_a_00373.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.46000000834465027,"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":120,"referenced_works":["https://openalex.org/W136732505","https://openalex.org/W658020064","https://openalex.org/W1544827683","https://openalex.org/W1826672328","https://openalex.org/W1956340063","https://openalex.org/W2101105183","https://openalex.org/W2110921901","https://openalex.org/W2112740777","https://openalex.org/W2119717200","https://openalex.org/W2120481102","https://openalex.org/W2130942839","https://openalex.org/W2133512280","https://openalex.org/W2153579005","https://openalex.org/W2154652894","https://openalex.org/W2158751803","https://openalex.org/W2182572585","https://openalex.org/W2250332012","https://openalex.org/W2250342921","https://openalex.org/W2250537810","https://openalex.org/W2251023345","https://openalex.org/W2251904119","https://openalex.org/W2341401723","https://openalex.org/W2507756961","https://openalex.org/W2606974598","https://openalex.org/W2612675303","https://openalex.org/W2758374095","https://openalex.org/W2788139998","https://openalex.org/W2793429163","https://openalex.org/W2807008868","https://openalex.org/W2888556271","https://openalex.org/W2889518897","https://openalex.org/W2890013547","https://openalex.org/W2890419630","https://openalex.org/W2891526960","https://openalex.org/W2892036296","https://openalex.org/W2896780650","https://openalex.org/W2896807716","https://openalex.org/W2913991091","https://openalex.org/W2945260553","https://openalex.org/W2947771965","https://openalex.org/W2951134195","https://openalex.org/W2951265142","https://openalex.org/W2952245931","https://openalex.org/W2962762898","https://openalex.org/W2962785754","https://openalex.org/W2962849707","https://openalex.org/W2962884827","https://openalex.org/W2962972512","https://openalex.org/W2962985882","https://openalex.org/W2963227052","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963545005","https://openalex.org/W2963607157","https://openalex.org/W2964016842","https://openalex.org/W2964285114","https://openalex.org/W2964308564","https://openalex.org/W2970419734","https://openalex.org/W2970785793","https://openalex.org/W2970830889","https://openalex.org/W2970892365","https://openalex.org/W2971032890","https://openalex.org/W2971034336","https://openalex.org/W2971274815","https://openalex.org/W2971289520","https://openalex.org/W2973379954","https://openalex.org/W2981456979","https://openalex.org/W2981852735","https://openalex.org/W2982399380","https://openalex.org/W2996403597","https://openalex.org/W3022685184","https://openalex.org/W3034188538","https://openalex.org/W3034383590","https://openalex.org/W3034715004","https://openalex.org/W3035628162","https://openalex.org/W3082115681","https://openalex.org/W3099766584","https://openalex.org/W3105678454","https://openalex.org/W3106234277","https://openalex.org/W3158986179","https://openalex.org/W6605449825","https://openalex.org/W6621906925","https://openalex.org/W6632455782","https://openalex.org/W6641064462","https://openalex.org/W6676057719","https://openalex.org/W6679434410","https://openalex.org/W6679436768","https://openalex.org/W6682631176","https://openalex.org/W6682691769","https://openalex.org/W6683252230","https://openalex.org/W6686204840","https://openalex.org/W6686386953","https://openalex.org/W6691492844","https://openalex.org/W6691642095","https://openalex.org/W6691896648","https://openalex.org/W6702998470","https://openalex.org/W6703902474","https://openalex.org/W6725207838","https://openalex.org/W6737479944","https://openalex.org/W6739901393","https://openalex.org/W6748217140","https://openalex.org/W6749743566","https://openalex.org/W6749986899","https://openalex.org/W6750851183","https://openalex.org/W6752183284","https://openalex.org/W6755207826","https://openalex.org/W6755478319","https://openalex.org/W6761205521","https://openalex.org/W6766803791","https://openalex.org/W6767186723","https://openalex.org/W6767858076","https://openalex.org/W6769311223","https://openalex.org/W6769627184","https://openalex.org/W6771915120","https://openalex.org/W6774455031","https://openalex.org/W6776957785","https://openalex.org/W6777215276","https://openalex.org/W6782465632","https://openalex.org/W6795224213","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W2154652894","https://openalex.org/W3159259047","https://openalex.org/W2996403597","https://openalex.org/W2101105183","https://openalex.org/W1544827683","https://openalex.org/W2971034336","https://openalex.org/W2888482885","https://openalex.org/W3034188538","https://openalex.org/W2982399380","https://openalex.org/W2963403868","https://openalex.org/W2963341956","https://openalex.org/W2606974598","https://openalex.org/W3099766584","https://openalex.org/W2963929190","https://openalex.org/W2889518897","https://openalex.org/W3035628162","https://openalex.org/W3035252911","https://openalex.org/W3034715004","https://openalex.org/W2962965405","https://openalex.org/W2962849707"],"abstract_inverted_index":{"Abstract":[0],"The":[1],"scarcity":[2],"of":[3,16,30,82,122,138,142,145],"comprehensive":[4,46],"up-to-date":[5],"studies":[6],"on":[7,88,148],"evaluation":[8,19,32,42,74,172,184],"metrics":[9,43,185],"for":[10,114,174],"text":[11,175],"summarization":[12,31,52,68,116,176],"and":[13,47,58,93,103,111,125,129,133,158],"the":[14,27,71,79,89,131,149],"lack":[15],"consensus":[17],"regarding":[18],"protocols":[20],"continue":[21],"to":[22],"inhibit":[23],"progress.":[24],"We":[25,161],"address":[26],"existing":[28],"shortcomings":[29],"methods":[33],"along":[34,55],"five":[35],"dimensions:":[36],"1)":[37],"we":[38,63,77,101,127],"re-evaluate":[39],"14":[40],"automatic":[41,73,123],"in":[44,96,136,182],"a":[45,97,105,119,169],"consistent":[48],"fashion":[49],"using":[50,70],"neural":[51],"model":[53,139],"outputs":[54],"with":[56,189],"expert":[57,156],"crowd-sourced":[59],"human":[60,143,190],"annotations;":[61],"2)":[62],"consistently":[64],"benchmark":[65],"23":[66],"recent":[67],"models":[69,86,117],"aforementioned":[72],"metrics;":[75,124],"3)":[76],"assemble":[78,128],"largest":[80,132],"collection":[81,141],"summaries":[83,147],"generated":[84],"by":[85,154],"trained":[87],"CNN/DailyMail":[90],"news":[91],"dataset":[92,152],"share":[94,104,130],"it":[95],"unified":[98,112],"format;":[99],"4)":[100],"implement":[102],"toolkit":[106],"that":[107,163,186],"provides":[108],"an":[109],"extensible":[110],"API":[113],"evaluating":[115],"across":[118],"broad":[120],"range":[121],"5)":[126],"most":[134],"diverse,":[135],"terms":[137],"types,":[140],"judgments":[144],"model-generated":[146],"CNN/Daily":[150],"Mail":[151],"annotated":[153],"both":[155],"judges":[157],"crowd-source":[159],"workers.":[160],"hope":[162],"this":[164],"work":[165],"will":[166],"help":[167],"promote":[168],"more":[170],"complete":[171],"protocol":[173],"as":[177,179],"well":[178],"advance":[180],"research":[181],"developing":[183],"better":[187],"correlate":[188],"judgments.":[191]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":18},{"year":2020,"cited_by_count":3}],"updated_date":"2026-07-01T08:55:40.977307","created_date":"2025-10-10T00:00:00"}