{"id":"https://openalex.org/W2740866094","doi":"https://doi.org/10.18653/v1/w17-1605","title":"Building Better Open-Source Tools to Support Fairness in Automated Scoring","display_name":"Building Better Open-Source Tools to Support Fairness in Automated Scoring","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2740866094","doi":"https://doi.org/10.18653/v1/w17-1605","mag":"2740866094"},"language":"en","primary_location":{"id":"doi:10.18653/v1/w17-1605","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w17-1605","pdf_url":"https://www.aclweb.org/anthology/W17-1605.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the First ACL Workshop on Ethics in Natural Language Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W17-1605.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015235744","display_name":"Nitin Madnani","orcid":"https://orcid.org/0000-0001-9354-6851"},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nitin Madnani","raw_affiliation_strings":["Educational Testing Service, Princeton, NJ","{nmadnani,aloukina,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, NJ","institution_ids":["https://openalex.org/I1341030882"]},{"raw_affiliation_string":"{nmadnani,aloukina,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064750543","display_name":"Anastassia Loukina","orcid":"https://orcid.org/0000-0002-8213-3362"},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anastassia Loukina","raw_affiliation_strings":["Educational Testing Service, Princeton, NJ","{nmadnani,aloukina,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, NJ","institution_ids":["https://openalex.org/I1341030882"]},{"raw_affiliation_string":"{nmadnani,aloukina,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026254988","display_name":"Alina von Davier","orcid":"https://orcid.org/0000-0002-5505-5311"},"institutions":[{"id":"https://openalex.org/I4210155253","display_name":"ACT","ror":"https://ror.org/05183v471","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210155253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alina von Davier","raw_affiliation_strings":["ACT, Iowa City, IA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ACT, Iowa City, IA","institution_ids":["https://openalex.org/I4210155253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030991878","display_name":"Jill Burstein","orcid":"https://orcid.org/0000-0001-7725-7574"},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jill Burstein","raw_affiliation_strings":["Educational Testing Service, Princeton, NJ","{nmadnani,aloukina,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, NJ","institution_ids":["https://openalex.org/I1341030882"]},{"raw_affiliation_string":"{nmadnani,aloukina,","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048408884","display_name":"Aoife Cahill","orcid":null},"institutions":[{"id":"https://openalex.org/I1341030882","display_name":"Educational Testing Service","ror":"https://ror.org/03b5q4637","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1341030882"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aoife Cahill","raw_affiliation_strings":["Educational Testing Service, Princeton, NJ","{nmadnani,aloukina,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Educational Testing Service, Princeton, NJ","institution_ids":["https://openalex.org/I1341030882"]},{"raw_affiliation_string":"{nmadnani,aloukina,","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":41,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"41","last_page":"52"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9472000002861023,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8127326369285583},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5851958394050598},{"id":"https://openalex.org/keywords/test-of-english-as-a-foreign-language","display_name":"Test of English as a Foreign Language","score":0.5524555444717407},{"id":"https://openalex.org/keywords/scoring-system","display_name":"Scoring system","score":0.5085238218307495},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.49240291118621826},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4744354784488678},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4622371792793274},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.4396364688873291},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.438178688287735},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4212425649166107},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3714487552642822},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.16845044493675232},{"id":"https://openalex.org/keywords/language-assessment","display_name":"Language assessment","score":0.14516159892082214},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1370658278465271},{"id":"https://openalex.org/keywords/mathematics-education","display_name":"Mathematics education","score":0.0988457202911377},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.08146876096725464}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8127326369285583},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5851958394050598},{"id":"https://openalex.org/C18563106","wikidata":"https://www.wikidata.org/wiki/Q487425","display_name":"Test of English as a Foreign Language","level":3,"score":0.5524555444717407},{"id":"https://openalex.org/C2984538763","wikidata":"https://www.wikidata.org/wiki/Q2260590","display_name":"Scoring system","level":2,"score":0.5085238218307495},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49240291118621826},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4744354784488678},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4622371792793274},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.4396364688873291},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.438178688287735},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4212425649166107},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3714487552642822},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.16845044493675232},{"id":"https://openalex.org/C166007726","wikidata":"https://www.wikidata.org/wiki/Q4536542","display_name":"Language assessment","level":2,"score":0.14516159892082214},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1370658278465271},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.0988457202911377},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.08146876096725464},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C141071460","wikidata":"https://www.wikidata.org/wiki/Q40821","display_name":"Surgery","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/w17-1605","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w17-1605","pdf_url":"https://www.aclweb.org/anthology/W17-1605.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the First ACL Workshop on Ethics in Natural Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/w17-1605","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w17-1605","pdf_url":"https://www.aclweb.org/anthology/W17-1605.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the First ACL Workshop on Ethics in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7099999785423279}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2740866094.pdf","grobid_xml":"https://content.openalex.org/works/W2740866094.grobid-xml"},"referenced_works_count":60,"referenced_works":["https://openalex.org/W28412257","https://openalex.org/W74346918","https://openalex.org/W1034637324","https://openalex.org/W1578803171","https://openalex.org/W1961345416","https://openalex.org/W1964537161","https://openalex.org/W1979364015","https://openalex.org/W1979769549","https://openalex.org/W1984114997","https://openalex.org/W2014352947","https://openalex.org/W2016195898","https://openalex.org/W2025719462","https://openalex.org/W2034131105","https://openalex.org/W2044244438","https://openalex.org/W2048411675","https://openalex.org/W2052086036","https://openalex.org/W2054745812","https://openalex.org/W2059141064","https://openalex.org/W2060423038","https://openalex.org/W2067895686","https://openalex.org/W2072182472","https://openalex.org/W2085182643","https://openalex.org/W2088070094","https://openalex.org/W2097246321","https://openalex.org/W2101234009","https://openalex.org/W2105845572","https://openalex.org/W2116984840","https://openalex.org/W2118426465","https://openalex.org/W2138519554","https://openalex.org/W2147427185","https://openalex.org/W2157928966","https://openalex.org/W2158240052","https://openalex.org/W2162670686","https://openalex.org/W2166454173","https://openalex.org/W2169958266","https://openalex.org/W2235875403","https://openalex.org/W2250653455","https://openalex.org/W2250817048","https://openalex.org/W2250952041","https://openalex.org/W2251191978","https://openalex.org/W2251472752","https://openalex.org/W2253640769","https://openalex.org/W2294237342","https://openalex.org/W2472803348","https://openalex.org/W2507708481","https://openalex.org/W2524301210","https://openalex.org/W2605970847","https://openalex.org/W2765277740","https://openalex.org/W2973803993","https://openalex.org/W2989885917","https://openalex.org/W3034730645","https://openalex.org/W3216779397","https://openalex.org/W4230965853","https://openalex.org/W4236581905","https://openalex.org/W4243407175","https://openalex.org/W4285190188","https://openalex.org/W4293091683","https://openalex.org/W4293091714","https://openalex.org/W4294101941","https://openalex.org/W4301911704"],"related_works":["https://openalex.org/W3146471590","https://openalex.org/W3156676763","https://openalex.org/W2549054388","https://openalex.org/W2397939076","https://openalex.org/W594209303","https://openalex.org/W2568692290","https://openalex.org/W4388343274","https://openalex.org/W2976001930","https://openalex.org/W3212654589","https://openalex.org/W2182738955"],"abstract_inverted_index":{"Automated":[0],"scoring":[1,36,56,96],"of":[2,21],"written":[3],"and":[4,28,54,83,91],"spoken":[5],"responses":[6],"is":[7,68],"an":[8],"NLP":[9,76],"application":[10],"that":[11,34,67,78],"can":[12],"significantly":[13],"impact":[14],"lives":[15],"especially":[16],"when":[17],"deployed":[18],"as":[19,25],"part":[20],"high-stakes":[22],"tests":[23],"such":[24,105],"the":[26,29,85],"GRE":[27],"TOEFL.":[30],"Ethical":[31],"considerations":[32],"require":[33],"automated":[35,55],"algorithms":[37],"treat":[38],"all":[39],"testtakers":[40],"fairly.":[41],"The":[42,62],"educational":[43],"measurement":[44],"community":[45],"has":[46],"done":[47],"significant":[48],"research":[49],"on":[50],"fairness":[51],"in":[52,94],"assessments":[53],"systems":[57],"must":[58],"incorporate":[59,80],"their":[60,95],"recommendations.":[61],"best":[63],"way":[64],"to":[65,75,88,103],"do":[66],"by":[69],"making":[70],"available":[71],"automated,":[72],"non-proprietary":[73],"tools":[74],"researchers":[77],"directly":[79],"these":[81],"recommendations":[82],"generate":[84],"analyses":[86],"needed":[87],"help":[89],"identify":[90],"resolve":[92],"biases":[93],"systems.":[97],"In":[98],"this":[99],"paper,":[100],"we":[101],"attempt":[102],"provide":[104],"a":[106],"solution.":[107]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":6}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}