{"id":"https://openalex.org/W2199528998","doi":null,"title":"Exploring the Reliability and Validity of Pilot Teacher Ratings in a Large California School District.","display_name":"Exploring the Reliability and Validity of Pilot Teacher Ratings in a Large California School District.","publication_year":2013,"publication_date":"2013-01-01","ids":{"openalex":"https://openalex.org/W2199528998","mag":"2199528998"},"language":"en","primary_location":{"id":"pmh:oai:escholarship.org/ark:/13030/qt0b63n98b","doi":null,"is_oa":true,"landing_page_url":"https://escholarship.org/uc/item/0b63n98b","pdf_url":"https://escholarship.org/uc/item/0b63n98b","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"etd"},"volume":null,"type":"dissertation","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://escholarship.org/uc/item/0b63n98b","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076979718","display_name":"Reino Makkonen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Reino Makkonen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5076979718"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13853","display_name":"Teacher Education and Assessments","score":0.5375999808311462,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13853","display_name":"Teacher Education and Assessments","score":0.5375999808311462,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13405","display_name":"Educational Assessment and Improvement","score":0.2888000011444092,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12952","display_name":"School Leadership and Teacher Performance","score":0.09210000187158585,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/teacher-evaluation","display_name":"teacher evaluation","score":0.9549999833106995},{"id":"https://openalex.org/keywords/rating-validity","display_name":"rating validity","score":0.6420000195503235},{"id":"https://openalex.org/keywords/rater-agreement","display_name":"rater agreement","score":0.7400000095367432},{"id":"https://openalex.org/keywords/number-of-raters","display_name":"number of raters","score":0.625},{"id":"https://openalex.org/keywords/generalizability-analysis","display_name":"generalizability analysis","score":0.6809999942779541},{"id":"https://openalex.org/keywords/teacher-effectiveness","display_name":"teacher effectiveness","score":0.40400001406669617},{"id":"https://openalex.org/keywords/teacher-reflection","display_name":"teacher reflection","score":0.4699999988079071},{"id":"https://openalex.org/keywords/california-school-districts","display_name":"California school districts","score":0.746999979019165},{"id":"https://openalex.org/keywords/teacher-accountability","display_name":"teacher accountability","score":0.5299999713897705},{"id":"https://openalex.org/keywords/student-test-scores","display_name":"student test scores","score":0.49300000071525574}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.7759916186332703},{"id":"https://openalex.org/C2777512617","wikidata":"https://www.wikidata.org/wiki/Q1783605","display_name":"Staffing","level":2,"score":0.722253143787384},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5644834637641907},{"id":"https://openalex.org/C2778137410","wikidata":"https://www.wikidata.org/wiki/Q2732820","display_name":"Government (linguistics)","level":2,"score":0.5327878594398499},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.48062795400619507},{"id":"https://openalex.org/C23131810","wikidata":"https://www.wikidata.org/wiki/Q818574","display_name":"Observational study","level":2,"score":0.47958841919898987},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.4601607322692871},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.43344414234161377},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.42263519763946533},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3553007245063782},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.1611914038658142},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.15716299414634705},{"id":"https://openalex.org/C159110408","wikidata":"https://www.wikidata.org/wiki/Q121176","display_name":"Nursing","level":1,"score":0.1387961208820343},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:escholarship.org/ark:/13030/qt0b63n98b","doi":null,"is_oa":true,"landing_page_url":"https://escholarship.org/uc/item/0b63n98b","pdf_url":"https://escholarship.org/uc/item/0b63n98b","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"etd"},{"id":"pmh:qt0b63n98b","doi":null,"is_oa":true,"landing_page_url":"http://www.escholarship.org/uc/item/0b63n98b","pdf_url":"http://www.escholarship.org/uc/item/0b63n98b","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Makkonen, Reino. (2013). Exploring the Reliability and Validity of Pilot Teacher Ratings in a Large California School District. UC Berkeley: Education. Retrieved from: http://www.escholarship.org/uc/item/0b63n98b","raw_type":"dissertation"},{"id":"mag:2199528998","doi":null,"is_oa":false,"landing_page_url":"https://escholarship.org/content/qt0b63n98b/qt0b63n98b.pdf?t=ny5vrh","pdf_url":null,"source":{"id":"https://openalex.org/S4306478302","display_name":"ProQuest LLC eBooks","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"ebook platform"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"ProQuest LLC eBooks","raw_type":null}],"best_oa_location":{"id":"pmh:oai:escholarship.org/ark:/13030/qt0b63n98b","doi":null,"is_oa":true,"landing_page_url":"https://escholarship.org/uc/item/0b63n98b","pdf_url":"https://escholarship.org/uc/item/0b63n98b","source":{"id":"https://openalex.org/S4306400115","display_name":"eScholarship (California Digital Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801248553","host_organization_name":"California Digital Library","host_organization_lineage":["https://openalex.org/I2801248553"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"etd"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.975600004196167}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2199528998.pdf","grobid_xml":"https://content.openalex.org/works/W2199528998.grobid-xml"},"referenced_works_count":59,"referenced_works":["https://openalex.org/W55883634","https://openalex.org/W145153723","https://openalex.org/W177318457","https://openalex.org/W280541720","https://openalex.org/W334502450","https://openalex.org/W1491613324","https://openalex.org/W1499284399","https://openalex.org/W1499826530","https://openalex.org/W1510953796","https://openalex.org/W1704942559","https://openalex.org/W1785956180","https://openalex.org/W1862185716","https://openalex.org/W1929056675","https://openalex.org/W1958015814","https://openalex.org/W1966671532","https://openalex.org/W1971665643","https://openalex.org/W1995722356","https://openalex.org/W2000653458","https://openalex.org/W2002470959","https://openalex.org/W2013651486","https://openalex.org/W2024509488","https://openalex.org/W2037789405","https://openalex.org/W2053154970","https://openalex.org/W2055257241","https://openalex.org/W2056040430","https://openalex.org/W2061546712","https://openalex.org/W2078800482","https://openalex.org/W2085259499","https://openalex.org/W2087877594","https://openalex.org/W2097437797","https://openalex.org/W2104882209","https://openalex.org/W2108034422","https://openalex.org/W2108181987","https://openalex.org/W2111443640","https://openalex.org/W2116253704","https://openalex.org/W2123812457","https://openalex.org/W2128169318","https://openalex.org/W2130631175","https://openalex.org/W2132573734","https://openalex.org/W2133527607","https://openalex.org/W2140000358","https://openalex.org/W2144706796","https://openalex.org/W2149506559","https://openalex.org/W2162140083","https://openalex.org/W2165653499","https://openalex.org/W2166610841","https://openalex.org/W2170095768","https://openalex.org/W2182711593","https://openalex.org/W2237598570","https://openalex.org/W2290081502","https://openalex.org/W2498989857","https://openalex.org/W2600046225","https://openalex.org/W2612851029","https://openalex.org/W2905010601","https://openalex.org/W2953140972","https://openalex.org/W2962747280","https://openalex.org/W3193532866","https://openalex.org/W3216779397","https://openalex.org/W4398227565"],"related_works":["https://openalex.org/W1017076799","https://openalex.org/W2472282007","https://openalex.org/W2911416380","https://openalex.org/W7033615186","https://openalex.org/W7097807431","https://openalex.org/W2261890334","https://openalex.org/W315723366","https://openalex.org/W2595374759","https://openalex.org/W3013881825","https://openalex.org/W2123812457"],"abstract_inverted_index":{"Many":[0,37],"states":[1],"and":[2,63,67,110,123,144,153,157,183,212,223,280,321,388,392,400,454,492,497,523,582,607,617,648,656,671,673,682],"school":[3,92,296,567],"districts":[4],"have":[5,41],"recently":[6],"instituted":[7],"revamped":[8],"teacher":[9,71,85,127],"evaluation":[10,86],"policies":[11],"in":[12,88,94,337,538,595,696],"response":[13,611],"to":[14,58,161,265,299,345,377,463,478,481,576,602,626],"incentives":[15],"from":[16,251,286,402,571],"the":[17,32,43,77,102,106,111,121,126,148,179,226,231,234,270,284,295,307,319,335,351,355,408,419,424,437,444,467,501,511,528,535,552,635,643,646,659,662,669,680],"federal":[18],"government":[19],"as":[20,22,118,120,578],"well":[21,119],"a":[23,81,89,194,242,448,559],"changing":[24],"political":[25],"climate":[26],"favoring":[27],"holding":[28],"teachers":[29,143,174,200,222,239,263,429,536,568,616,672],"accountable":[30],"for":[31,190,233,500,591,615,620],"performance":[33,47,177],"of":[34,38,45,65,80,125,141,186,197,230,237,245,253,334,385,410,421,426,439,451,506,513,521,534,558,563,637,642],"their":[35,176,254,440,452,464,493],"students.":[36],"these":[39,262,514,686],"overhauls":[40],"mandated":[42],"incorporation":[44],"multiple":[46],"indicators":[48],"--":[49,69,273,279,531],"often":[50],"including":[51,170],"rubric-based":[52],"classroom":[53,457,489],"observation":[54,128,258,356],"scores,":[55],"estimated":[56],"contributions":[57],"student":[59,494],"test":[60],"score":[61],"outcomes,":[62],"surveys":[64,433],"students":[66],"parents":[68],"into":[70],"evaluations.":[72],"This":[73],"three-paper":[74],"dissertation":[75],"explores":[76],"pilot":[78,133,142,227,339,398,428,445,455,487,508,518,529,554,663],"implementation":[79,399],"new":[82,103,150,180],"standards-based":[83],"multiple-measure":[84],"system":[87,104],"large":[90],"California":[91],"district":[93,660],"2011/12.":[95],"It":[96],"examines":[97],"both":[98,221,252,257],"participants'":[99],"views":[100],"about":[101,175],"(particularly":[105],"challenges":[107,218],"they":[108,114,470],"faced":[109],"early":[112,166],"outcomes":[113,167],"felt":[115,435],"were":[116,168,215,374,460,484,599,652,666,675],"achieved),":[117],"reliability":[122,379,413],"validity":[124],"ratings":[129,232,459,491,496,544],"that":[130,137,163,202,261,281,331,364,436,473],"resulted":[131],"during":[132,225,269,443,527,661],"implementation.":[134],"Results":[135],"indicated":[136,260,330],"this":[138,203,403,507],"self-selected":[139],"group":[140,368,425,562],"administrators":[145,198,224,570,674],"generally":[146,434],"appreciated":[147],"district's":[149],"teaching":[151,181],"framework":[152,182],"pre/post-observation":[154],"conferencing":[155],"process,":[156],"participants":[158],"also":[159,622],"tended":[160,264,298,575],"report":[162],"certain":[164],"key":[165,217],"achieved,":[169],"increased":[171],"reflection":[172],"by":[173,220,306,547,658],"against":[178],"better":[184],"understandings":[185],"teachers'":[187,308,340,456,488],"individual":[188],"needs":[189],"instructional":[191],"support":[192],"(although":[193],"higher":[195,268],"proportion":[196],"than":[199,303,363,417],"reported":[201],"latter":[204],"outcome":[205],"was":[206,316,343,361,525,556],"achieved).":[207],"Time":[208],"constraints,":[209],"staffing":[210],"shortfalls":[211],"technology":[213],"problems":[214],"all":[216],"cited":[219],"year.":[228,503],"Analyses":[229],"small":[235],"sample":[236,555],"participating":[238,338,427,605],"who":[240,324,430,569,639],"received":[241],"complete":[243],"set":[244],"observational":[246],"focus":[247],"element":[248],"(item)":[249],"scores":[250,285,342,499],"raters":[255,288,323],"across":[256,282],"cycles":[259,283],"be":[266,300,476,694],"scored":[267,325],"second":[271,287,322],"cycle":[272,357],"although":[274],"such":[275],"improvement":[276],"wasn't":[277],"universal":[278],"(who":[289],"typically":[290],"did":[291,632],"not":[292,461,633,690],"work":[293],"at":[294],"site)":[297],"slightly":[301],"lower":[302],"those":[304,638],"awarded":[305],"supervising":[309],"site":[310],"administrator.":[311],"But":[312],"ultimately,":[313],"good":[314],"agreement":[315],"evident":[317,485],"between":[318,486],"primary":[320],"common":[326],"teachers.":[327],"Generalizability":[328],"analyses":[329],"approximately":[332,532],"two-thirds":[333],"variation":[336],"total":[341],"attributable":[344],"systematic":[346],"differences":[347],"among":[348],"teachers,":[349],"while":[350],"variability":[352],"associated":[353,365],"with":[354,366,679],"(approximately":[358,369],"25":[359],"percent)":[360],"larger":[362],"rater":[367,386],"6":[370],"percent).":[371],"These":[372],"results":[373,401,687],"then":[375],"used":[376],"forecast":[378],"coefficients":[380],"based":[381,395],"on":[382,397],"different":[383],"combinations":[384],"groups":[387],"observed":[389],"lessons":[390],"(cycles),":[391],"suggested":[393],"that,":[394],"solely":[396],"particular":[404],"analysis":[405],"sample,":[406],"varying":[407,418],"number":[409,420],"observations":[411,438],"influenced":[412],"estimates":[414],"far":[415],"more":[416,629],"observers.":[422],"Finally,":[423,645],"completed":[431],"end-of-year":[432],"practice":[441],"conducted":[442],"year":[446,530],"represented":[447],"valid":[449],"measure":[450],"effectiveness,":[453],"observation-based":[458,490],"related":[462],"ethnicity":[465],"or":[466],"grade":[468],"span":[469],"taught":[471],"(factors":[472],"should":[474],"theoretically":[475],"unrelated":[477],"performance).":[479],"Low":[480],"moderate":[482],"correlations":[483],"survey":[495,610],"value-added":[498],"2011/12":[502],"The":[504,517],"uniqueness":[505],"context":[509],"restricts":[510],"generalizability":[512],"findings,":[515],"however.":[516],"consisted":[519],"primarily":[520],"volunteers,":[522],"there":[524],"attrition":[526],"one-third":[533],"trained":[537],"fall":[539],"2011":[540],"never":[541],"had":[542],"any":[543,697],"entered":[545],"online":[546],"an":[548],"observer.":[549],"In":[550],"turn,":[551],"final":[553],"comprised":[557],"self":[560],"selected":[561],"experienced,":[564],"mostly":[565],"elementary":[566],"case":[572],"study":[573,651],"sites":[574],"characterize":[577],"particularly":[579],"hard":[580],"working":[581],"high":[583],"performing.":[584],"Moreover,":[585],"our":[586,608,624],"research":[587],"team's":[588],"limited":[589],"capacity":[590],"qualitative":[592],"data":[593],"collection":[594],"spring":[596],"2012":[597],"(we":[598],"only":[600],"able":[601],"visit":[603],"five":[604],"schools)":[606],"low":[609],"rates":[612],"(52":[613],"percent":[614,619],"54":[618],"administrators)":[621],"limit":[623],"ability":[625],"generalize":[627],"findings":[628],"broadly.":[630],"We":[631],"hear":[634],"perspectives":[636],"dropped":[640],"out":[641],"pilot.":[644],"tools":[647,670],"processes":[649,681],"under":[650],"still":[653,667],"being":[654],"revised":[655],"fine-tuned":[657],"year;":[664],"observers":[665],"learning":[668],"just":[676],"becoming":[677],"familiar":[678],"measures.":[683],"All":[684],"told,":[685],"likely":[688],"do":[689],"reflect":[691],"what":[692],"will":[693],"found":[695],"eventual":[698],"full-scale":[699],"roll":[700],"out.":[701]},"counts_by_year":[],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}