{"id":"https://openalex.org/W2533973643","doi":null,"title":"Issues and challenges in current generalizability theory applications in rated measurement","display_name":"Issues and challenges in current generalizability theory applications in rated measurement","publication_year":2014,"publication_date":"2014-09-16","ids":{"openalex":"https://openalex.org/W2533973643","mag":"2533973643"},"language":"en","primary_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/50656","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2142/50656","pdf_url":"https://www.ideals.illinois.edu/items/50767/bitstreams/147520/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ideals.illinois.edu/items/50767/bitstreams/147520/data.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024614998","display_name":"Chih\u2010Kai Lin","orcid":"https://orcid.org/0000-0002-8032-3066","observed_orcids":["https://orcid.org/0000-0002-8032-3066"]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Chih\u2010Kai Lin","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-8032-3066","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5024614998"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":93},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.98580002784729,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.98580002784729,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.003700000001117587,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12884","display_name":"Educational Assessment and Pedagogy","score":0.0019000000320374966,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"generalizability theory","score":0.9980000257492065},{"id":"https://openalex.org/keywords/human-raters","display_name":"human raters","score":0.6840000152587891},{"id":"https://openalex.org/keywords/performance-based-assessment","display_name":"performance-based assessment","score":0.722000002861023},{"id":"https://openalex.org/keywords/data-sparsity","display_name":"data sparsity","score":0.4959999918937683},{"id":"https://openalex.org/keywords/variance-components","display_name":"variance components","score":0.6340000033378601},{"id":"https://openalex.org/keywords/measurement-reliability","display_name":"measurement reliability","score":0.5580000281333923},{"id":"https://openalex.org/keywords/standard-setting","display_name":"standard setting","score":0.5709999799728394},{"id":"https://openalex.org/keywords/content-validation","display_name":"content validation","score":0.9419999718666077},{"id":"https://openalex.org/keywords/english-proficiency","display_name":"English proficiency","score":0.8330000042915344},{"id":"https://openalex.org/keywords/rating-methods","display_name":"rating methodology","score":0.9419999718666077},{"id":"https://openalex.org/keywords/rater-variability","display_name":"rater variability","score":0.3319999873638153}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.7908416986465454},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.6261801719665527},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3883432149887085},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3496609032154083},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3378620445728302},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3347298204898834},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.2075652778148651},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16987672448158264},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.09293216466903687},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.08959856629371643}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:www.ideals.illinois.edu:2142/50656","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2142/50656","pdf_url":"https://www.ideals.illinois.edu/items/50767/bitstreams/147520/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2533973643","doi":null,"is_oa":false,"landing_page_url":"https://www.ideals.illinois.edu/handle/2142/50656","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:www.ideals.illinois.edu:2142/50656","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/2142/50656","pdf_url":"https://www.ideals.illinois.edu/items/50767/bitstreams/147520/data.pdf","source":{"id":"https://openalex.org/S4306401358","display_name":"Illinois Digital Environment for Access to Learning and Scholarship (University of Illinois at Urbana-Champaign)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801919071","host_organization_name":"University of Illinois System","host_organization_lineage":["https://openalex.org/I2801919071"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2533973643.pdf","grobid_xml":"https://content.openalex.org/works/W2533973643.grobid-xml"},"referenced_works_count":65,"referenced_works":["https://openalex.org/W189026518","https://openalex.org/W656565818","https://openalex.org/W657332045","https://openalex.org/W1507497905","https://openalex.org/W1530997910","https://openalex.org/W1533893528","https://openalex.org/W1562811233","https://openalex.org/W1586414640","https://openalex.org/W1588286827","https://openalex.org/W1592263067","https://openalex.org/W1958015814","https://openalex.org/W1966952698","https://openalex.org/W1968961518","https://openalex.org/W1969363772","https://openalex.org/W1972314731","https://openalex.org/W1973490241","https://openalex.org/W1975316857","https://openalex.org/W1988338581","https://openalex.org/W1988500224","https://openalex.org/W1994610994","https://openalex.org/W1998312332","https://openalex.org/W1999912466","https://openalex.org/W2006032025","https://openalex.org/W2006444775","https://openalex.org/W2016730986","https://openalex.org/W2020435271","https://openalex.org/W2025610145","https://openalex.org/W2028773126","https://openalex.org/W2034294879","https://openalex.org/W2036410403","https://openalex.org/W2039016361","https://openalex.org/W2043458675","https://openalex.org/W2043544836","https://openalex.org/W2062137446","https://openalex.org/W2062695010","https://openalex.org/W2067899227","https://openalex.org/W2070147348","https://openalex.org/W2072296938","https://openalex.org/W2077649418","https://openalex.org/W2081597998","https://openalex.org/W2086338484","https://openalex.org/W2090591874","https://openalex.org/W2095043073","https://openalex.org/W2097993933","https://openalex.org/W2098655575","https://openalex.org/W2099827822","https://openalex.org/W2107229891","https://openalex.org/W2111803260","https://openalex.org/W2116642507","https://openalex.org/W2130631175","https://openalex.org/W2138477560","https://openalex.org/W2141403362","https://openalex.org/W2142257270","https://openalex.org/W2144415364","https://openalex.org/W2144848743","https://openalex.org/W2157133049","https://openalex.org/W2159131964","https://openalex.org/W2171376992","https://openalex.org/W2316025483","https://openalex.org/W2320386680","https://openalex.org/W2470288897","https://openalex.org/W2501011868","https://openalex.org/W2787869423","https://openalex.org/W3141391145","https://openalex.org/W3216779397"],"related_works":["https://openalex.org/W202822135","https://openalex.org/W4392028131","https://openalex.org/W2049693959","https://openalex.org/W2161089488","https://openalex.org/W3045048812","https://openalex.org/W7095310874","https://openalex.org/W2605393382","https://openalex.org/W91601597","https://openalex.org/W2933516508","https://openalex.org/W1923248611"],"abstract_inverted_index":{"The":[0,228,285,406,644],"current":[1],"dissertation":[2,198],"looks":[3],"into":[4],"issues":[5,806,835,845],"and":[6,52,64,73,99,158,188,203,216,225,245,294,328,356,426,478,509,541,578,592,617,631,674,683,694,701,734,755,778,799,839,928],"challenges":[7,785],"regarding":[8,846],"the":[9,105,110,114,126,131,148,184,191,238,255,269,308,334,343,347,395,403,415,431,441,487,491,510,515,522,525,549,555,587,605,614,652,657,672,679,691,697,708,740,757,774,776,791,797,809,818,824,847,876,879,887],"use":[10,454,848],"of":[11,79,96,101,107,133,150,186,194,219,251,290,346,380,417,490,512,521,535,551,554,575,595,607,629,638,654,659,671,681,716,730,748,765,793,812,820,832,849,878,894],"generalizability":[12],"theory":[13,16,169,221,537],"or":[14,669],"G":[15,168,220,536],"(Brennan,":[17],"2001;":[18],"Cronbach,":[19],"Gleser,":[20],"Nanda,":[21],"&":[22,26],"Rajaratnam,":[23],"1972;":[24],"Shavelson":[25],"Webb,":[27],"1991)":[28],"in":[29,37,57,84,89,93,125,147,222,241,282,371,390,414,430,485,506,514,538,546,597,641,656,662,719,728,744,761,773,786,790,807,817,836,883,898],"rated":[30,59,135,223,278,663,870],"measurement":[31,40,60,71,76,108,136,151,156,187,596,639,682,684,852,884,920],"given":[32,860],"by":[33,321,333,783,804],"human":[34],"raters.":[35,552],"Contexts":[36],"which":[38,113,272,444],"such":[39,460,922],"prevails":[41],"include,":[42],"but":[43],"are":[44,61,87,163,273,366,448,600,622,686,781,802,857],"not":[45],"limited":[46],"to,":[47],"performance-based":[48,283],"assessments,":[49],"standard":[50,593],"settings,":[51],"content":[53,428,810],"validation":[54],"studies.":[55],"Inherent":[56],"expert":[58],"potential":[62],"systematic":[63,80],"random":[65,120,161,326],"variations":[66,92,153],"that":[67,130,175,236,362,496,611,739,756,861,906],"can":[68],"contribute":[69],"to":[70,178,189,214,275,315,341,368,443,450,468,499,531,542,548,559,604,624,632,650,678,711,853],"errors,":[72],"thereby":[74],"affect":[75],"reliability.":[77],"Examples":[78],"variability":[81,121,146,389],"(i.e.,":[82,109],"facets":[83,103,157],"G-theory":[85,270,588,660,704,788,837,855,888,900],"terminology)":[86],"differences":[88,182],"rater":[90,94,286,396],"severity/leniency,":[91],"interpretations":[95],"scoring":[97],"criteria,":[98],"interactions":[100],"these":[102,305],"with":[104,155,160,457,569,602,926],"objects":[106,149,185,680],"subjects":[111],"on":[112,261,300,475,482,613,871],"intended":[115],"construct":[116],"is":[117,137,411,726,742,759,890,902],"measured),":[118],"whereas":[119],"reflects":[122],"unexpected":[123],"fluctuations":[124],"rating":[127,309,404],"process.":[128],"Given":[129],"utility":[132],"any":[134],"contingent":[138],"upon":[139],"its":[140],"reliability,":[141],"analytical":[142,173,266],"tools":[143],"for":[144,634,724,746,841],"disentangling":[145],"from":[152,263,565],"associated":[154,159,456,925],"errors":[162,594,640],"necessary.":[164],"To":[165],"this":[166],"end,":[167],"provides":[170,247],"a":[171,200,226,242,248,301,312,317,325,351,501,581,635,830,862,872],"powerful":[172],"framework":[174,889],"allows":[176],"investigators":[177],"tease":[179],"out":[180,413],"true":[181],"among":[183],"assess":[190],"relative":[192],"magnitude":[193],"construct-irrelevant":[195],"variability.":[196],"This":[197,338,463],"follows":[199],"multi-paper":[201],"approach":[202,467],"includes":[204],"six":[205],"chapters,":[206],"including":[207],"an":[208,233,357,519,626,891,904],"introduction,":[209],"four":[210],"individual":[211],"papers":[212,240,780,801],"pertaining":[213,677],"theoretical":[215],"applied":[217],"investigations":[218],"measurement,":[224,795],"conclusion.":[227],"introduction":[229],"(Chapter":[230,258,409,528,647,826],"1)":[231],"sketches":[232],"overarching":[234],"theme":[235],"situates":[237],"separate":[239],"thematic":[243],"unity":[244],"also":[246,480,903],"brief":[249],"summary":[250],"each":[252],"paper.":[253],"Next,":[254],"first":[256,777],"paper":[257,339,408,464,527,646,692,709],"2)":[259],"reports":[260],"findings":[262],"comparing":[264],"two":[265,348],"methods,":[267],"under":[268,886],"framework,":[271,589],"designed":[274],"analyze":[276],"sparse":[277,313],"data":[279],"commonly":[280],"observed":[281],"assessments.":[284],"method":[287,310,376,397],"identifies":[288],"blocks":[289],"fully":[291,318],"crossed":[292,319],"sub-datasets":[293],"then":[295,329],"estimates":[296,330,379,401],"variance":[297,331,381],"components":[298,332],"based":[299,474],"weighted":[302],"average":[303],"across":[304],"sub-datasets,":[306],"while":[307,796],"forces":[311],"dataset":[314],"be":[316,369,500],"one":[320,466,763,919],"conceptualizing":[322],"ratings":[323,392,599],"as":[324,461,580,829,923],"facet":[327],"usual":[335],"crossed-design":[336],"procedures.":[337],"aims":[340,710],"compare":[342],"estimation":[344],"precision":[345],"methods":[349],"via":[350],"Monte":[352],"Carlo":[353],"simulation":[354],"study":[355],"empirical":[358],"study.":[359],"Results":[360,494,609,737],"show":[361,610,738],"when":[363,384,667],"all":[364,670],"raters":[365,386,927],"expected":[367,449],"homogeneous":[370],"their":[372,391,507,598,642],"score":[373],"variability,":[374],"either":[375],"has":[377],"good":[378],"components.":[382],"However,":[383],"some":[385,668,833],"exhibit":[387],"more":[388,399,908,917],"than":[393,402,918],"others,":[394],"yields":[398],"precise":[400],"method.":[405],"second":[407,523,582,798],"3)":[410],"carried":[412],"context":[416,658],"examining":[418],"correspondence":[419,436,472,539,567],"between":[420,503,699],"English":[421,445,579,814],"language":[422,446,453,560,583,815,821],"proficiency":[423],"(ELP)":[424],"standards":[425,429],"academic":[427,452,458],"US":[432],"K-12":[433],"setting.":[434],"Such":[435],"studies":[437,568],"provide":[438],"information":[439],"about":[440],"extent":[442],"learners":[447,816],"encounter":[451],"closely":[455],"disciplines,":[459],"mathematics.":[462],"describes":[465],"conducting":[469],"ELP":[470],"standards-to-standards":[471],"research":[473,540,909],"reviewer":[476,483,504,544,590],"judgments,":[477],"it":[479],"touches":[481],"consistency":[484,505],"judging":[486],"cognitive":[488,556],"complexity":[489,557],"target":[492,516],"standards.":[493,517],"suggest":[495],"there":[497],"seems":[498],"relationship":[502],"judgments":[508],"level":[511,628],"specificity":[513],"As":[518,689,769],"extension":[520],"paper,":[524],"third":[526,800],"4)":[529],"seeks":[530],"advance":[532,651],"new":[533],"applications":[534,661,789,838],"examine":[543],"reliability":[545,591,630],"relation":[547],"numbers":[550,606],"Ratings":[553],"germane":[558],"performance":[561,863,913],"indicators":[562],"were":[563],"collected":[564],"28":[566],"over":[570],"700":[571],"trained":[572],"reviewers,":[573],"consisting":[574],"content-area":[576],"experts":[577],"(ESL)":[584],"specialists.":[585],"Under":[586],"evaluated":[601,727],"respect":[603],"reviewers.":[608],"depending":[612],"particular":[615],"grades":[616],"subject":[618],"areas,":[619],"3-6":[620],"reviewers":[621],"needed":[623],"achieve":[625],"acceptable":[627],"control":[633],"reasonable":[636],"amount":[637],"ratings.":[643],"fourth":[645,779],"5)":[648],"attempts":[649],"discussion":[653,831],"nonadditivity":[655,718,725,751,897],"measurement.":[664],"Nonadditivity":[665],"occurs":[666],"main":[673],"interaction":[675,767],"effects,":[676],"facet(s),":[685],"significantly":[687],"correlated.":[688],"such,":[690],"analytically":[693],"empirically":[695],"illustrates":[696],"distinction":[698],"additive":[700],"nonadditive":[702,766],"one-facet":[703],"models.":[705],"In":[706],"addition,":[707],"explore":[712],"existing":[713],"statistical":[714,735],"procedures":[715],"detecting":[717],"data.":[720],"Tukey's":[721],"single-degree-freedom":[722],"test":[723,741,758,864],"terms":[729],"Type":[731],"I":[732,753],"error":[733],"power.":[736],"satisfactory":[743],"controlling":[745],"occurrences":[747],"erroneously":[749],"identifying":[750,762],"(Type":[752],"error)":[754],"successful":[760],"type":[764],"(power).":[768],"will":[770],"become":[771],"clear":[772],"dissertation,":[775],"motivated":[782,803],"methodological":[784],"advancing":[787],"field":[792,819],"educational":[794],"validity":[805],"assessing":[808],"knowledge":[811],"young":[813],"testing.":[822],"Finally,":[823,896],"conclusion":[825],"6)":[827],"functions":[828],"unsolved":[834,892],"ideas":[840],"future":[842],"research.":[843,895],"First,":[844],"many-facet":[850],"Rasch":[851],"complement":[854],"analysis":[856],"discussed.":[858],"Second,":[859],"usually":[865],"involves":[866],"examinee":[867],"responses":[868],"being":[869],"discrete":[873,880],"ordinal":[874,881],"scale,":[875],"consideration":[877],"nature":[882],"variables":[885],"area":[893,905],"multi-faceted":[899],"models":[901],"deserves":[907],"efforts":[910],"because":[911],"most":[912],"tests":[914],"would":[915],"entail":[916],"facet,":[921],"those":[924],"tasks.":[929]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2016-10-28T00:00:00"}