{"id":"https://openalex.org/W1483044931","doi":null,"title":"Comparing rating scales and preference judgements in language evaluation","display_name":"Comparing rating scales and preference judgements in language evaluation","publication_year":2010,"publication_date":"2010-07-07","ids":{"openalex":"https://openalex.org/W1483044931","mag":"1483044931"},"language":"en","primary_location":{"id":"pmh:oai:pure.atira.dk:publications/06c3feb3-4a66-4ee8-a413-df8932e1d6b3","doi":null,"is_oa":true,"landing_page_url":"https://research.brighton.ac.uk/en/publications/06c3feb3-4a66-4ee8-a413-df8932e1d6b3","pdf_url":"http://dl.acm.org/citation.cfm?id=1873743","source":{"id":"https://openalex.org/S4306401758","display_name":"University of Brighton Repository (University of Brighton)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I71637028","host_organization_name":"University of Brighton","host_organization_lineage":["https://openalex.org/I71637028"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Belz , A & Kow , E 2010 , Comparing rating scales and preference judgements in language evaluation . in Proceedings of the 6th International Language Generation Congerence (INLG'10) . Association for Computational Linguistics , Stroudsburg, PA, USA , pp. 7-15 , Proceedings of the 6th International Language Generation Congerence (INLG'10) , 1/01/10 . https://doi.org/10.1.1.167.7542","raw_type":"contributionToPeriodical"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://dl.acm.org/citation.cfm?id=1873743","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010118897","display_name":"Anja Belz","orcid":"https://orcid.org/0000-0002-0552-8096","observed_orcids":["https://orcid.org/0000-0002-0552-8096"]},"institutions":[{"id":"https://openalex.org/I71637028","display_name":"University of Brighton","ror":"https://ror.org/04kp2b655","country_code":"GB","type":"education","lineage":["https://openalex.org/I71637028"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Anja Belz","raw_affiliation_strings":["University of Brighton, Brighton-UK#TAB#"],"raw_orcid":"https://orcid.org/0000-0002-0552-8096","affiliations":[{"raw_affiliation_string":"University of Brighton, Brighton-UK#TAB#","institution_ids":["https://openalex.org/I71637028"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5053478598","display_name":"Eric Kow","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I71637028","display_name":"University of Brighton","ror":"https://ror.org/04kp2b655","country_code":"GB","type":"education","lineage":["https://openalex.org/I71637028"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Eric Kow","raw_affiliation_strings":["University of Brighton, Brighton-UK#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Brighton, Brighton-UK#TAB#","institution_ids":["https://openalex.org/I71637028"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I71637028"],"apc_list":null,"apc_paid":null,"fwci":1.2081,"has_fulltext":true,"cited_by_count":26,"citation_normalized_percentile":{"value":0.8030682,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"7","last_page":"15"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.8119999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.8119999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.049800001084804535,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.01730000041425228,"subfield":{"id":"https://openalex.org/subfields/3310","display_name":"Linguistics and Language"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rating-scales","display_name":"rating scales","score":0.6100000143051147},{"id":"https://openalex.org/keywords/inter-examiner-agreement","display_name":"inter-examiner agreement","score":0.7559999823570251},{"id":"https://openalex.org/keywords/ordinal-data","display_name":"ordinal data","score":0.75},{"id":"https://openalex.org/keywords/text-fluency","display_name":"text fluency","score":0.9710000157356262},{"id":"https://openalex.org/keywords/text-clarity","display_name":"text clarity","score":0.9919999837875366}],"concepts":[{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.7052617073059082},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.690647304058075},{"id":"https://openalex.org/C2777146004","wikidata":"https://www.wikidata.org/wiki/Q14949826","display_name":"CLARITY","level":2,"score":0.6159484386444092},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.6104480028152466},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5966989398002625},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5817192196846008},{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.5787155628204346},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5740636587142944},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5533509254455566},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.5308243632316589},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.506761372089386},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.43998655676841736},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.43375542759895325},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4306222200393677},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.37589508295059204},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36736559867858887},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3498244285583496},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.26754772663116455},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.07951003313064575},{"id":"https://openalex.org/C44870925","wikidata":"https://www.wikidata.org/wiki/Q37547","display_name":"Astrophysics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:pure.atira.dk:publications/06c3feb3-4a66-4ee8-a413-df8932e1d6b3","doi":null,"is_oa":true,"landing_page_url":"https://research.brighton.ac.uk/en/publications/06c3feb3-4a66-4ee8-a413-df8932e1d6b3","pdf_url":"http://dl.acm.org/citation.cfm?id=1873743","source":{"id":"https://openalex.org/S4306401758","display_name":"University of Brighton Repository (University of Brighton)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I71637028","host_organization_name":"University of Brighton","host_organization_lineage":["https://openalex.org/I71637028"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Belz , A & Kow , E 2010 , Comparing rating scales and preference judgements in language evaluation . in Proceedings of the 6th International Language Generation Congerence (INLG'10) . Association for Computational Linguistics , Stroudsburg, PA, USA , pp. 7-15 , Proceedings of the 6th International Language Generation Congerence (INLG'10) , 1/01/10 . https://doi.org/10.1.1.167.7542","raw_type":"contributionToPeriodical"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.167.7542","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.167.7542","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.nltg.brighton.ac.uk/home/Anja.Belz/Publications/belz-kow-inlg10-long.pdf","raw_type":"text"},{"id":"mag:1483044931","doi":null,"is_oa":false,"landing_page_url":"https://aclanthology.org/W10-4201/","pdf_url":null,"source":{"id":"https://openalex.org/S4306419715","display_name":"International Conference on Natural Language Generation","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Natural Language Generation","raw_type":null}],"best_oa_location":{"id":"pmh:oai:pure.atira.dk:publications/06c3feb3-4a66-4ee8-a413-df8932e1d6b3","doi":null,"is_oa":true,"landing_page_url":"https://research.brighton.ac.uk/en/publications/06c3feb3-4a66-4ee8-a413-df8932e1d6b3","pdf_url":"http://dl.acm.org/citation.cfm?id=1873743","source":{"id":"https://openalex.org/S4306401758","display_name":"University of Brighton Repository (University of Brighton)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I71637028","host_organization_name":"University of Brighton","host_organization_lineage":["https://openalex.org/I71637028"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Belz , A & Kow , E 2010 , Comparing rating scales and preference judgements in language evaluation . in Proceedings of the 6th International Language Generation Congerence (INLG'10) . Association for Computational Linguistics , Stroudsburg, PA, USA , pp. 7-15 , Proceedings of the 6th International Language Generation Congerence (INLG'10) , 1/01/10 . https://doi.org/10.1.1.167.7542","raw_type":"contributionToPeriodical"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1483044931.pdf","grobid_xml":"https://content.openalex.org/works/W1483044931.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W28870373","https://openalex.org/W1489409710","https://openalex.org/W1763968285","https://openalex.org/W2030416917","https://openalex.org/W2054496757","https://openalex.org/W2075746878","https://openalex.org/W2076258857","https://openalex.org/W2119874156","https://openalex.org/W2142018618","https://openalex.org/W2142112143"],"related_works":["https://openalex.org/W2990527403","https://openalex.org/W7166836052","https://openalex.org/W2107587102","https://openalex.org/W4416034445","https://openalex.org/W2158905336","https://openalex.org/W2599082534","https://openalex.org/W4385572789","https://openalex.org/W7214977693","https://openalex.org/W4416036248","https://openalex.org/W3046022111"],"abstract_inverted_index":{"Rating-scale":[0],"evaluations":[1],"are":[2,7,38],"common":[3],"in":[4,75,140],"NLP,":[5],"but":[6],"problematic":[8],"for":[9,19,43,93,135],"a":[10,78,105,112,129,141],"range":[11],"of":[12,67,70,77,85,99,102,118,132,144],"reasons,":[13],"e.g.":[14],"they":[15],"can":[16],"be":[17,27],"unintuitive":[18],"evaluators,":[20],"inter-evaluator":[21],"agreement":[22],"and":[23,29,91,108,125,128],"self-consistency":[24,124],"tend":[25],"to":[26,35],"low,":[28],"the":[30,36,64,109,115,119],"parametric":[31],"statistics":[32],"commonly":[33],"applied":[34],"results":[37],"not":[39],"generally":[40],"considered":[41],"appropriate":[42],"ordinal":[44],"data.In":[45],"this":[46],"paper,":[47],"we":[48],"compare":[49],"rating":[50],"scales":[51],"with":[52],"an":[53],"alternative":[54],"evaluation":[55,86],"paradigm,":[56],"preferencestrength":[57],"judgement":[58],"experiments":[59,87,103,120],"(PJEs),":[60],"where":[61,97],"evaluators":[62],"have":[63,121],"simpler":[65],"task":[66],"deciding":[68],"which":[69],"two":[71],"texts":[72],"is":[73,104,111],"better":[74,122],"terms":[76],"given":[79],"quality":[80],"criterion.We":[81],"present":[82],"three":[83],"pairs":[84],"assessing":[88],"text":[89],"fluency":[90],"clarity":[92],"different":[94],"data":[95],"sets,":[96],"one":[98],"each":[100],"pair":[101],"rating-scale":[106],"experiment,":[107],"other":[110],"PJE.We":[113],"find":[114],"PJE":[116],"versions":[117],"evaluator":[123],"interevaluator":[126],"agreement,":[127],"larger":[130,142],"proportion":[131],"variation":[133],"accounted":[134],"by":[136],"system":[137],"differences,":[138],"resulting":[139],"number":[143],"significant":[145],"differences":[146],"being":[147],"found.":[148]},"counts_by_year":[{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":6},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":2},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2026-11-03T10:50:19.383937","created_date":"2016-06-24T00:00:00"}