{"id":"https://openalex.org/W2804545603","doi":"https://doi.org/10.18653/v1/n18-1021","title":"Automated Essay Scoring in the Presence of Biased Ratings","display_name":"Automated Essay Scoring in the Presence of Biased Ratings","alternate_titles":[],"publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W2804545603","doi":"https://doi.org/10.18653/v1/n18-1021","mag":"2804545603"},"language":"en","primary_location":{"id":"doi:10.18653/v1/n18-1021","doi":"https://doi.org/10.18653/v1/n18-1021","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/n18-1021","pdf_url":"https://www.aclweb.org/anthology/N18-1021.pdf","source":{"id":"https://openalex.org/S4363608774","display_name":"Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W2804545603"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://www.aclweb.org/anthology/N18-1021.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039335917","display_name":"Evelin Carvalho Freire de Amorim","orcid":"https://orcid.org/0000-0003-1343-939X","observed_orcids":["https://orcid.org/0000-0003-1343-939X"]},"institutions":[{"id":"https://openalex.org/I110200422","display_name":"Universidade Federal de Minas Gerais","ror":"https://ror.org/0176yjw32","country_code":"BR","type":"education","lineage":["https://openalex.org/I110200422"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Evelin Amorim","raw_affiliation_strings":["Computer Science Department UFMG Brazil","Linguistics Department UFMG Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department UFMG Brazil","institution_ids":["https://openalex.org/I110200422"]},{"raw_affiliation_string":"Linguistics Department UFMG Brazil","institution_ids":["https://openalex.org/I110200422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018448041","display_name":"M\u00e1rcia Can\u00e7ado","orcid":"https://orcid.org/0000-0002-4159-3661","observed_orcids":["https://orcid.org/0000-0002-4159-3661"]},"institutions":[{"id":"https://openalex.org/I110200422","display_name":"Universidade Federal de Minas Gerais","ror":"https://ror.org/0176yjw32","country_code":"BR","type":"education","lineage":["https://openalex.org/I110200422"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Marcia Can\u00e7ado","raw_affiliation_strings":["Computer Science Department UFMG Brazil","Linguistics Department UFMG Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department UFMG Brazil","institution_ids":["https://openalex.org/I110200422"]},{"raw_affiliation_string":"Linguistics Department UFMG Brazil","institution_ids":["https://openalex.org/I110200422"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086714399","display_name":"Adriano Veloso","orcid":"https://orcid.org/0000-0002-9177-4954","observed_orcids":["https://orcid.org/0000-0002-9177-4954"]},"institutions":[{"id":"https://openalex.org/I110200422","display_name":"Universidade Federal de Minas Gerais","ror":"https://ror.org/0176yjw32","country_code":"BR","type":"education","lineage":["https://openalex.org/I110200422"]}],"countries":["BR"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Adriano Veloso","raw_affiliation_strings":["Computer Science Department UFMG Brazil","Linguistics Department UFMG Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department UFMG Brazil","institution_ids":["https://openalex.org/I110200422"]},{"raw_affiliation_string":"Linguistics Department UFMG Brazil","institution_ids":["https://openalex.org/I110200422"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I110200422"],"apc_list":null,"apc_paid":null,"fwci":3.047,"has_fulltext":true,"cited_by_count":106,"reviewed_by_count":0,"citation_normalized_percentile":{"value":0.94026555,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"229","last_page":"237"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.4503999948501587,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.4503999948501587,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14025","display_name":"Educational Technology and Assessment","score":0.2084999978542328,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.02969999983906746,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-essay-scoring","display_name":"automated essay scoring","score":0.9990000128746033},{"id":"https://openalex.org/keywords/rater-bias","display_name":"rater bias","score":1.0},{"id":"https://openalex.org/keywords/rater-effects","display_name":"rater effects","score":0.6110000014305115},{"id":"https://openalex.org/keywords/bias-quantification","display_name":"bias quantification","score":0.5899999737739563},{"id":"https://openalex.org/keywords/handcrafted-features","display_name":"handcrafted features","score":0.6710000038146973}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5715253353118896},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33237409591674805}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/n18-1021","doi":"https://doi.org/10.18653/v1/n18-1021","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/n18-1021","pdf_url":"https://www.aclweb.org/anthology/N18-1021.pdf","source":{"id":"https://openalex.org/S4363608774","display_name":"Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/n18-1021","doi":"https://doi.org/10.18653/v1/n18-1021","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/n18-1021","pdf_url":"https://www.aclweb.org/anthology/N18-1021.pdf","source":{"id":"https://openalex.org/S4363608774","display_name":"Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1678887205","display_name":null,"funder_award_id":"MCT/CNPq 573871/2008- 6","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G3029329602","display_name":"MASWEB - MODELOS, ALGORITMOS E SISTEMAS PARA WEB","funder_award_id":"APQ-01400-14","funder_id":"https://openalex.org/F4320322980","funder_display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais"},{"id":"https://openalex.org/G3450651771","display_name":null,"funder_award_id":"FAPEMIG/PRONEX APQ-01400-14","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G6217110605","display_name":null,"funder_award_id":"FAPEMIG/PRONEX APQ-01400-14","funder_id":"https://openalex.org/F4320322980","funder_display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais"},{"id":"https://openalex.org/G874191044","display_name":null,"funder_award_id":"573871/2008-6","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"}],"funders":[{"id":"https://openalex.org/F4320322025","display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","ror":"https://ror.org/03swz6y49"},{"id":"https://openalex.org/F4320322980","display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais","ror":"https://ror.org/00nc55f03"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2804545603.pdf","grobid_xml":"https://content.openalex.org/works/W2804545603.grobid-xml"},"referenced_works_count":41,"referenced_works":["https://openalex.org/W73269325","https://openalex.org/W600456550","https://openalex.org/W658020064","https://openalex.org/W1508985630","https://openalex.org/W1629317078","https://openalex.org/W1911711584","https://openalex.org/W1969967679","https://openalex.org/W2008161323","https://openalex.org/W2016377072","https://openalex.org/W2029181554","https://openalex.org/W2053154970","https://openalex.org/W2075358683","https://openalex.org/W2087385735","https://openalex.org/W2101234009","https://openalex.org/W2105897558","https://openalex.org/W2145451908","https://openalex.org/W2145770094","https://openalex.org/W2146849125","https://openalex.org/W2153579005","https://openalex.org/W2155823788","https://openalex.org/W2158240052","https://openalex.org/W2187089797","https://openalex.org/W2250372169","https://openalex.org/W2250887565","https://openalex.org/W2250952041","https://openalex.org/W2251191978","https://openalex.org/W2294725250","https://openalex.org/W2407119034","https://openalex.org/W2440757793","https://openalex.org/W2510438573","https://openalex.org/W2564237127","https://openalex.org/W2567547739","https://openalex.org/W2574905972","https://openalex.org/W2740592764","https://openalex.org/W2741201359","https://openalex.org/W2757310600","https://openalex.org/W2903849486","https://openalex.org/W2986538395","https://openalex.org/W3213767167","https://openalex.org/W4253060983","https://openalex.org/W4292157289"],"related_works":["https://openalex.org/W4210468047","https://openalex.org/W4382567213","https://openalex.org/W3038362714","https://openalex.org/W3029258614","https://openalex.org/W2767465286","https://openalex.org/W4409493991","https://openalex.org/W2251191978","https://openalex.org/W2154414741","https://openalex.org/W4412889410","https://openalex.org/W7017413971"],"abstract_inverted_index":{"Studies":[0],"in":[1,38,60,99,125],"Social":[2],"Sciences":[3],"have":[4],"revealed":[5],"that":[6,27,118],"when":[7],"people":[8],"evaluate":[9],"someone":[10],"else,":[11],"their":[12,16,29,84,103,113],"evaluations":[13,30],"often":[14],"reflect":[15],"biases.As":[17],"a":[18,77],"result,":[19],"rater":[20,58,67,109,119,134],"bias":[21,68,110,120,135],"may":[22,32],"introduce":[23],"highly":[24],"subjective":[25],"factors":[26],"make":[28],"inaccurate.This":[31],"affect":[33],"automated":[34,70,126],"essay":[35,51,71,127],"scoring":[36,160],"models":[37,43,137],"many":[39],"ways,":[40],"as":[41],"these":[42],"are":[44],"typically":[45],"designed":[46],"to":[47,101,107,132,144],"model":[48],"(potentially":[49],"biased)":[50],"raters.While":[52],"there":[53],"is":[54],"sizeable":[55],"literature":[56],"on":[57,112,139],"effects":[59],"general":[61],"settings,":[62],"it":[63],"remains":[64],"unknown":[65],"how":[66],"affects":[69,136],"scoring.To":[72],"this":[73],"end,":[74],"we":[75,116,142],"present":[76,105],"new":[78],"annotated":[79],"corpus":[80,91],"containing":[81],"essays":[82,151],"and":[83,115],"respective":[85],"scores.Different":[86],"from":[87],"existing":[88],"corpora,":[89],"our":[90],"also":[92],"contains":[93],"comments":[94],"provided":[95],"by":[96,149],"the":[97,130,146,159],"raters":[98],"order":[100],"ground":[102],"scores.We":[104],"features":[106],"quantify":[108],"based":[111,138],"comments,":[114],"found":[117],"plays":[121],"an":[122],"important":[123],"role":[124],"scoring.We":[128],"investigated":[129],"extent":[131],"which":[133],"hand-crafted":[140],"features.Finally,":[141],"propose":[143],"rectify":[145],"training":[147],"set":[148],"removing":[150],"associated":[152],"with":[153],"potentially":[154],"biased":[155],"scores":[156],"while":[157],"learning":[158],"model.":[161]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2804545603","counts_by_year":[{"year":2026,"cited_by_count":16},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":14},{"year":2020,"cited_by_count":15},{"year":2019,"cited_by_count":12}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2018-06-01T00:00:00"}
