{"id":"https://openalex.org/W2804545603","doi":"https://doi.org/10.18653/v1/n18-1021","title":"Automated Essay Scoring in the Presence of Biased Ratings","display_name":"Automated Essay Scoring in the Presence of Biased Ratings","publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W2804545603","doi":"https://doi.org/10.18653/v1/n18-1021","mag":"2804545603"},"language":"en","primary_location":{"id":"doi:10.18653/v1/n18-1021","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/n18-1021","pdf_url":"https://www.aclweb.org/anthology/N18-1021.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference of the North American Chapter of\n          the Association for Computational Linguistics: Human Language\n          Technologies, Volume 1 (Long Papers)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/N18-1021.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039335917","display_name":"Evelin Amorim","orcid":"https://orcid.org/0000-0003-1343-939X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Evelin Amorim","raw_affiliation_strings":["Computer Science Department UFMG Brazil","Linguistics Department UFMG Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department UFMG Brazil","institution_ids":[]},{"raw_affiliation_string":"Linguistics Department UFMG Brazil","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018448041","display_name":"M\u00e1rcia Can\u00e7ado","orcid":"https://orcid.org/0000-0002-4159-3661"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marcia Can\u00e7ado","raw_affiliation_strings":["Linguistics Department UFMG Brazil","Computer Science Department UFMG Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Linguistics Department UFMG Brazil","institution_ids":[]},{"raw_affiliation_string":"Computer Science Department UFMG Brazil","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086714399","display_name":"Adriano Veloso","orcid":"https://orcid.org/0000-0002-9177-4954"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Adriano Veloso","raw_affiliation_strings":["Linguistics Department UFMG Brazil","Computer Science Department UFMG Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Linguistics Department UFMG Brazil","institution_ids":[]},{"raw_affiliation_string":"Computer Science Department UFMG Brazil","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.9281,"has_fulltext":true,"cited_by_count":80,"citation_normalized_percentile":{"value":0.97444027,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"229","last_page":"237"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9544000029563904,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9433000087738037,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5715253353118896},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33237409591674805}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5715253353118896},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33237409591674805}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/n18-1021","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/n18-1021","pdf_url":"https://www.aclweb.org/anthology/N18-1021.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference of the North American Chapter of\n          the Association for Computational Linguistics: Human Language\n          Technologies, Volume 1 (Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/n18-1021","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/n18-1021","pdf_url":"https://www.aclweb.org/anthology/N18-1021.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference of the North American Chapter of\n          the Association for Computational Linguistics: Human Language\n          Technologies, Volume 1 (Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3029329602","display_name":null,"funder_award_id":"APQ-01400-14","funder_id":"https://openalex.org/F4320322980","funder_display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais"},{"id":"https://openalex.org/G3450651771","display_name":null,"funder_award_id":"FAPEMIG/PRONEX APQ-01400-14","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G4442413068","display_name":null,"funder_award_id":"573871/2008-6","funder_id":"https://openalex.org/F4320322980","funder_display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais"},{"id":"https://openalex.org/G5997353801","display_name":null,"funder_award_id":"MCT/CNPq 573871/2008-6","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G6217110605","display_name":null,"funder_award_id":"FAPEMIG/PRONEX APQ-01400-14","funder_id":"https://openalex.org/F4320322980","funder_display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais"},{"id":"https://openalex.org/G874191044","display_name":null,"funder_award_id":"573871/2008-6","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"}],"funders":[{"id":"https://openalex.org/F4320322025","display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","ror":"https://ror.org/03swz6y49"},{"id":"https://openalex.org/F4320322980","display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais","ror":"https://ror.org/00nc55f03"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2804545603.pdf","grobid_xml":"https://content.openalex.org/works/W2804545603.grobid-xml"},"referenced_works_count":41,"referenced_works":["https://openalex.org/W73269325","https://openalex.org/W600456550","https://openalex.org/W658020064","https://openalex.org/W1508985630","https://openalex.org/W1911711584","https://openalex.org/W1969967679","https://openalex.org/W2008161323","https://openalex.org/W2016377072","https://openalex.org/W2029181554","https://openalex.org/W2053154970","https://openalex.org/W2075358683","https://openalex.org/W2087385735","https://openalex.org/W2101234009","https://openalex.org/W2105897558","https://openalex.org/W2145451908","https://openalex.org/W2145770094","https://openalex.org/W2146849125","https://openalex.org/W2153579005","https://openalex.org/W2155823788","https://openalex.org/W2158240052","https://openalex.org/W2187089797","https://openalex.org/W2250372169","https://openalex.org/W2250887565","https://openalex.org/W2250952041","https://openalex.org/W2251191978","https://openalex.org/W2294725250","https://openalex.org/W2407119034","https://openalex.org/W2510438573","https://openalex.org/W2564237127","https://openalex.org/W2567547739","https://openalex.org/W2574905972","https://openalex.org/W2740592764","https://openalex.org/W2741201359","https://openalex.org/W2757310600","https://openalex.org/W2986538395","https://openalex.org/W3098654368","https://openalex.org/W3162099472","https://openalex.org/W3213767167","https://openalex.org/W4253060983","https://openalex.org/W4285719527","https://openalex.org/W4294170691"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2382290278","https://openalex.org/W2350741829","https://openalex.org/W2130043461","https://openalex.org/W2530322880","https://openalex.org/W1596801655"],"abstract_inverted_index":{"Studies":[0],"in":[1,40,63,104,131],"Social":[2],"Sciences":[3],"have":[4],"revealed":[5],"that":[6,28,124],"when":[7],"people":[8],"evaluate":[9],"someone":[10],"else,":[11],"their":[12,16,30,88,108,119],"evaluations":[13,31],"often":[14],"reflect":[15],"biases.":[17],"As":[18],"a":[19,81],"result,":[20],"rater":[21,61,70,115,125,141],"bias":[22,71,116,126,142],"may":[23,34],"introduce":[24],"highly":[25],"subjective":[26],"factors":[27],"make":[29],"inaccurate.":[32],"This":[33],"affect":[35],"automated":[36,73,132],"essay":[37,53,74,133],"scoring":[38,168],"models":[39,45,144],"many":[41],"ways,":[42],"as":[43],"these":[44],"are":[46],"typically":[47],"designed":[48],"to":[49,106,113,139,152],"model":[50],"(potentially":[51],"biased)":[52],"raters.":[54],"While":[55],"there":[56],"is":[57],"sizeable":[58],"literature":[59],"on":[60,118,146],"effects":[62],"general":[64],"settings,":[65],"it":[66],"remains":[67],"unknown":[68],"how":[69],"affects":[72,143],"scoring.":[75,134],"To":[76],"this":[77],"end,":[78],"we":[79,122,150],"present":[80,111],"new":[82],"annotated":[83],"corpus":[84,96],"containing":[85],"essays":[86,159],"and":[87,121],"respective":[89],"scores.":[90,109],"Different":[91],"from":[92],"existing":[93],"corpora,":[94],"our":[95],"also":[97],"contains":[98],"comments":[99],"provided":[100],"by":[101,157],"the":[102,137,154,167],"raters":[103],"order":[105],"ground":[107],"We":[110,135],"features":[112],"quantify":[114],"based":[117,145],"comments,":[120],"found":[123],"plays":[127],"an":[128],"important":[129],"role":[130],"investigated":[136],"extent":[138],"which":[140],"hand-crafted":[147],"features.":[148],"Finally,":[149],"propose":[151],"rectify":[153],"training":[155],"set":[156],"removing":[158],"associated":[160],"with":[161],"potentially":[162],"biased":[163],"scores":[164],"while":[165],"learning":[166],"model.":[169]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":11},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":14},{"year":2020,"cited_by_count":13},{"year":2019,"cited_by_count":13},{"year":2018,"cited_by_count":1}],"updated_date":"2026-06-27T08:28:00.272161","created_date":"2025-10-10T00:00:00"}