{"id":"https://openalex.org/W4281252138","doi":"https://doi.org/10.21449/ijate.993805","title":"Comparison of Inter-Rater Reliability Techniques in Performance-Based Assessment","display_name":"Comparison of Inter-Rater Reliability Techniques in Performance-Based Assessment","publication_year":2022,"publication_date":"2022-05-21","ids":{"openalex":"https://openalex.org/W4281252138","doi":"https://doi.org/10.21449/ijate.993805"},"language":"en","primary_location":{"id":"doi:10.21449/ijate.993805","doi":"https://doi.org/10.21449/ijate.993805","is_oa":true,"landing_page_url":"https://doi.org/10.21449/ijate.993805","pdf_url":"https://dergipark.org.tr/en/download/article-file/1968072","source":{"id":"https://openalex.org/S2738319323","display_name":"International Journal of Assessment Tools in Education","issn_l":"2148-7456","issn":["2148-7456"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","erih-plus","poland-20","russia-white-list-3","tr-dizin"],"host_organization":"https://openalex.org/P4310318394","host_organization_name":"International Journal of Assessment Tools in Education","host_organization_lineage":["https://openalex.org/P4310318394"],"host_organization_lineage_names":["International Journal of Assessment Tools in Education"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Assessment Tools in Education","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://dergipark.org.tr/en/download/article-file/1968072","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074866444","display_name":"Sinem ARSLAN MANCAR","orcid":"https://orcid.org/0000-0002-2031-2189","observed_orcids":["https://orcid.org/0000-0002-2031-2189"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sinem ARSLAN MANCAR","raw_affiliation_strings":["Activate Learning-Reading College","Independent Researcher"],"raw_orcid":"https://orcid.org/0000-0002-2031-2189","affiliations":[{"raw_affiliation_string":"Activate Learning-Reading College","institution_ids":[]},{"raw_affiliation_string":"Independent Researcher","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030858300","display_name":"Hamide Deniz G\u00fcllero\u011flu","orcid":"https://orcid.org/0000-0001-6995-8223","observed_orcids":["https://orcid.org/0000-0001-6995-8223"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hamide Deniz G\u00dcLLERO\u011eLU","raw_affiliation_strings":["Graduate School of Educational Sciences"],"raw_orcid":"https://orcid.org/0000-0001-6995-8223","affiliations":[{"raw_affiliation_string":"Graduate School of Educational Sciences","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3152,"has_fulltext":true,"cited_by_count":12,"citation_normalized_percentile":{"value":0.61386139,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":99},"biblio":{"volume":"9","issue":"2","first_page":"515","last_page":"533"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.3991999924182892,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.3991999924182892,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.36329999566078186,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.13500000536441803,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":1.0},{"id":"https://openalex.org/keywords/performance-based-assessment","display_name":"performance-based assessment","score":0.9150000214576721},{"id":"https://openalex.org/keywords/number-of-raters","display_name":"number of raters","score":0.8460000157356262},{"id":"https://openalex.org/keywords/kappa","display_name":"kappa","score":0.5950000286102295},{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"generalizability theory","score":0.8420000076293945},{"id":"https://openalex.org/keywords/classical-test-theory","display_name":"classical test theory","score":0.7379999756813049},{"id":"https://openalex.org/keywords/rubric-based-assessment","display_name":"rubric-based assessment","score":0.6140000224113464},{"id":"https://openalex.org/keywords/international-baccalaureate-diploma-programme","display_name":"International Baccalaureate Diploma Programme","score":0.953000009059906}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9252228140830994},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8989713788032532},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.7074652910232544},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7009367942810059},{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.6048657894134521},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5296288132667542},{"id":"https://openalex.org/C140390113","wikidata":"https://www.wikidata.org/wiki/Q389391","display_name":"Classical test theory","level":4,"score":0.5021255016326904},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.49051254987716675},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.37675929069519043},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.35134583711624146},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.32870304584503174},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.2963809370994568},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.28955626487731934},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2726261019706726},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.25098341703414917},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.23308545351028442},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.21449/ijate.993805","doi":"https://doi.org/10.21449/ijate.993805","is_oa":true,"landing_page_url":"https://doi.org/10.21449/ijate.993805","pdf_url":"https://dergipark.org.tr/en/download/article-file/1968072","source":{"id":"https://openalex.org/S2738319323","display_name":"International Journal of Assessment Tools in Education","issn_l":"2148-7456","issn":["2148-7456"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","erih-plus","poland-20","russia-white-list-3","tr-dizin"],"host_organization":"https://openalex.org/P4310318394","host_organization_name":"International Journal of Assessment Tools in Education","host_organization_lineage":["https://openalex.org/P4310318394"],"host_organization_lineage_names":["International Journal of Assessment Tools in Education"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Assessment Tools in Education","raw_type":"journal-article"},{"id":"pmh:oai:dergipark.org.tr:article/993805","doi":null,"is_oa":true,"landing_page_url":"https://dergipark.org.tr/tr/pub/ijate/issue/69546/993805","pdf_url":"https://dergipark.org.tr/tr/pub/ijate/issue/69546/993805","source":{"id":"https://openalex.org/S4306401840","display_name":"DergiPark (Istanbul University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67581229","host_organization_name":"Istanbul University","host_organization_lineage":["https://openalex.org/I67581229"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:doaj.org/article:f5f2429cd11748a897b34a490aae1898","doi":null,"is_oa":false,"landing_page_url":"https://doaj.org/article/f5f2429cd11748a897b34a490aae1898","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"International Journal of Assessment Tools in Education, Vol 9, Iss 2, Pp 515-533 (2022)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.21449/ijate.993805","doi":"https://doi.org/10.21449/ijate.993805","is_oa":true,"landing_page_url":"https://doi.org/10.21449/ijate.993805","pdf_url":"https://dergipark.org.tr/en/download/article-file/1968072","source":{"id":"https://openalex.org/S2738319323","display_name":"International Journal of Assessment Tools in Education","issn_l":"2148-7456","issn":["2148-7456"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","erih-plus","poland-20","russia-white-list-3","tr-dizin"],"host_organization":"https://openalex.org/P4310318394","host_organization_name":"International Journal of Assessment Tools in Education","host_organization_lineage":["https://openalex.org/P4310318394"],"host_organization_lineage_names":["International Journal of Assessment Tools in Education"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Assessment Tools in Education","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4281252138.pdf","grobid_xml":"https://content.openalex.org/works/W4281252138.grobid-xml"},"referenced_works_count":36,"referenced_works":["https://openalex.org/W77069522","https://openalex.org/W1444168786","https://openalex.org/W1514060233","https://openalex.org/W1516058724","https://openalex.org/W1517687210","https://openalex.org/W1530609927","https://openalex.org/W1548183903","https://openalex.org/W1600588650","https://openalex.org/W1650611486","https://openalex.org/W1893371726","https://openalex.org/W1975879668","https://openalex.org/W1980060142","https://openalex.org/W1986197238","https://openalex.org/W1986568529","https://openalex.org/W2009972843","https://openalex.org/W2034059165","https://openalex.org/W2039065426","https://openalex.org/W2051692807","https://openalex.org/W2053154970","https://openalex.org/W2060958129","https://openalex.org/W2065594816","https://openalex.org/W2164777277","https://openalex.org/W2310307435","https://openalex.org/W2480159269","https://openalex.org/W2599521805","https://openalex.org/W2610338946","https://openalex.org/W2782777952","https://openalex.org/W2903120359","https://openalex.org/W3021916629","https://openalex.org/W3023738774","https://openalex.org/W4232103669","https://openalex.org/W4238804190","https://openalex.org/W4241775175","https://openalex.org/W4247689549","https://openalex.org/W4301140379","https://openalex.org/W6677517115"],"related_works":["https://openalex.org/W2081855353","https://openalex.org/W4414736938","https://openalex.org/W2288200086","https://openalex.org/W7162267154","https://openalex.org/W3164725399","https://openalex.org/W2581818947","https://openalex.org/W3045048812","https://openalex.org/W2161089488","https://openalex.org/W2962056896","https://openalex.org/W1972341850"],"abstract_inverted_index":{"The":[0,32,95,124],"aim":[1],"of":[2,10,13,54,64,97,116,136,143,150],"this":[3,49],"study":[4,100],"is":[5,161],"to":[6,43,110,163],"analyse":[7],"the":[8,11,17,45,62,65,86,98,114,117,122,134,137,141,148,151,154,157,167,170],"importance":[9],"number":[12,142],"raters":[14,74,144,155],"and":[15,28,34,71,104,139],"compare":[16],"results":[18,96],"obtained":[19],"by":[20,61,85],"techniques":[21,37,108],"based":[22,38,126],"on":[23,39,127],"Classical":[24],"Test":[25],"Theory":[26,129],"(CTT)":[27],"Generalizability":[29],"(G)":[30],"Theory.":[31],"Kappa":[33,103],"Krippendorff":[35,105],"alpha":[36,106],"CTT":[40,99],"were":[41],"used":[42,80],"determine":[44],"inter-rater":[46],"reliability.":[47],"In":[48],"descriptive":[50],"research":[51],"data":[52,132],"consists":[53],"twenty":[55],"individual":[56],"investigation":[57],"performance":[58],"reports":[59],"prepared":[60],"learners":[63],"International":[66,87],"Baccalaureate":[67,88],"Diploma":[68],"Programme":[69],"(IBDP)":[70],"also":[72,146],"five":[73],"who":[75],"rated":[76],"these":[77],"reports.":[78],"Raters":[79],"an":[81],"analytical":[82],"rubric":[83],"developed":[84],"Organization":[89],"(IBO)":[90],"as":[91],"a":[92],"scoring":[93],"tool.":[94],"show":[101],"that":[102,159],"statistical":[107],"failed":[109],"provide":[111],"information":[112],"about":[113,133],"sources":[115,135],"errors":[118,138],"causing":[119],"incompatibility":[120],"in":[121,166,169],"criteria.":[123],"studies":[125],"G":[128],"provided":[130],"comprehensive":[131],"increasing":[140],"would":[145],"increase":[147],"reliability":[149],"values.":[152],"However,":[153],"raised":[156],"idea":[158],"it":[160],"important":[162],"develop":[164],"descriptors":[165],"criteria":[168],"rubric.":[171]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2022-05-23T00:00:00"}