{"id":"https://openalex.org/W2772829373","doi":"https://doi.org/10.1111/jedm.12157","title":"An Experimental Study of the Internal Consistency of Judgments Made in Bookmark Standard Setting","display_name":"An Experimental Study of the Internal Consistency of Judgments Made in Bookmark Standard Setting","publication_year":2017,"publication_date":"2017-12-01","ids":{"openalex":"https://openalex.org/W2772829373","doi":"https://doi.org/10.1111/jedm.12157","mag":"2772829373"},"language":"en","primary_location":{"id":"doi:10.1111/jedm.12157","doi":"https://doi.org/10.1111/jedm.12157","is_oa":false,"landing_page_url":"https://doi.org/10.1111/jedm.12157","pdf_url":null,"source":{"id":"https://openalex.org/S30638350","display_name":"Journal of Educational Measurement","issn_l":"0022-0655","issn":["0022-0655","1745-3984"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-c","erih-plus","ki-jl-2","norway-2","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Educational Measurement","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070469227","display_name":"Brian E. Clauser","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1293309919","display_name":"National Board of Medical Examiners","ror":"https://ror.org/04tsmra86","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1293309919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Brian E. Clauser","raw_affiliation_strings":["National Board of Medical Examiners"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Board of Medical Examiners","institution_ids":["https://openalex.org/I1293309919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102852506","display_name":"Peter Baldwin","orcid":"https://orcid.org/0000-0003-3472-0172","observed_orcids":["https://orcid.org/0000-0003-3472-0172"]},"institutions":[{"id":"https://openalex.org/I1293309919","display_name":"National Board of Medical Examiners","ror":"https://ror.org/04tsmra86","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1293309919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter Baldwin","raw_affiliation_strings":["National Board of Medical Examiners"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Board of Medical Examiners","institution_ids":["https://openalex.org/I1293309919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082641946","display_name":"Melissa J. Margolis","orcid":"https://orcid.org/0000-0002-6548-7273","observed_orcids":["https://orcid.org/0000-0002-6548-7273"]},"institutions":[{"id":"https://openalex.org/I1293309919","display_name":"National Board of Medical Examiners","ror":"https://ror.org/04tsmra86","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1293309919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Melissa J. Margolis","raw_affiliation_strings":["National Board of Medical Examiners"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Board of Medical Examiners","institution_ids":["https://openalex.org/I1293309919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081318319","display_name":"Janet Mee","orcid":"https://orcid.org/0000-0002-9983-1343","observed_orcids":["https://orcid.org/0000-0002-9983-1343"]},"institutions":[{"id":"https://openalex.org/I1293309919","display_name":"National Board of Medical Examiners","ror":"https://ror.org/04tsmra86","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1293309919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Janet Mee","raw_affiliation_strings":["National Board of Medical Examiners"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Board of Medical Examiners","institution_ids":["https://openalex.org/I1293309919"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062789748","display_name":"Marcia L. Winward","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1293309919","display_name":"National Board of Medical Examiners","ror":"https://ror.org/04tsmra86","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1293309919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marcia Winward","raw_affiliation_strings":["National Board of Medical Examiners"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Board of Medical Examiners","institution_ids":["https://openalex.org/I1293309919"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1293309919"],"apc_list":{"value":3040,"currency":"USD","value_usd":3040},"apc_paid":null,"fwci":1.2117,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.82420915,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":97},"biblio":{"volume":"54","issue":"4","first_page":"481","last_page":"497"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.9663000106811523,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.9663000106811523,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12884","display_name":"Educational Assessment and Pedagogy","score":0.01590000092983246,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13853","display_name":"Teacher Education and Assessments","score":0.004100000020116568,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/internal-consistency","display_name":"internal consistency","score":0.7940000295639038},{"id":"https://openalex.org/keywords/cutoff-values","display_name":"cut-off values","score":0.8740000128746033},{"id":"https://openalex.org/keywords/content-experts","display_name":"content experts","score":0.703000009059906},{"id":"https://openalex.org/keywords/criterion-referenced-testing","display_name":"criterion-referenced assessment","score":0.546999990940094}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7570226192474365},{"id":"https://openalex.org/C3018868096","wikidata":"https://www.wikidata.org/wiki/Q2693233","display_name":"Internal consistency","level":3,"score":0.6993089914321899},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6814839839935303},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5984994173049927},{"id":"https://openalex.org/C51082289","wikidata":"https://www.wikidata.org/wiki/Q4113800","display_name":"Internal validity","level":2,"score":0.41389212012290955},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.3693537414073944},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.33465611934661865},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.24880215525627136},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.17760199308395386},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.13028013706207275},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.10892367362976074},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07433590292930603},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1111/jedm.12157","doi":"https://doi.org/10.1111/jedm.12157","is_oa":false,"landing_page_url":"https://doi.org/10.1111/jedm.12157","pdf_url":null,"source":{"id":"https://openalex.org/S30638350","display_name":"Journal of Educational Measurement","issn_l":"0022-0655","issn":["0022-0655","1745-3984"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-c","erih-plus","ki-jl-2","norway-2","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Educational Measurement","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"display_name":"Peace, justice, and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.47999998927116394}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W612083133","https://openalex.org/W1519000222","https://openalex.org/W1567130159","https://openalex.org/W1704942559","https://openalex.org/W1981353909","https://openalex.org/W1996829812","https://openalex.org/W2006859681","https://openalex.org/W2013058208","https://openalex.org/W2026593185","https://openalex.org/W2052126401","https://openalex.org/W2053203354","https://openalex.org/W2058154574","https://openalex.org/W2069384544","https://openalex.org/W2086492164","https://openalex.org/W2089954256","https://openalex.org/W2107229891","https://openalex.org/W2117687944","https://openalex.org/W2140729693","https://openalex.org/W2148573653","https://openalex.org/W2505213109","https://openalex.org/W2797177778","https://openalex.org/W4246821341"],"related_works":["https://openalex.org/W2910533113","https://openalex.org/W2089954256","https://openalex.org/W2135209109","https://openalex.org/W154875592","https://openalex.org/W7098905383","https://openalex.org/W2086492164","https://openalex.org/W3118241496","https://openalex.org/W2519465547","https://openalex.org/W2521805334","https://openalex.org/W2142054799"],"abstract_inverted_index":{"Abstract":[0],"Validating":[1],"performance":[2],"standards":[3],"is":[4,41],"challenging":[5],"and":[6],"complex.":[7],"Because":[8],"of":[9,46,56,70,96,123,136],"the":[10,54,57,68,94,106,114,121,124,134,141,145],"difficulties":[11],"associated":[12],"with":[13],"collecting":[14],"evidence":[15,25,30,45],"related":[16,26],"to":[17,27,139],"external":[18],"criteria,":[19],"validity":[20],"arguments":[21],"rely":[22],"heavily":[23],"on":[24,75],"internal":[28,101],"criteria\u2014especially":[29],"that":[31,44,109],"expert":[32],"judgments":[33,142],"are":[34],"internally":[35,83],"consistent.":[36],"Given":[37],"its":[38],"importance,":[39],"it":[40],"somewhat":[42],"surprising":[43],"this":[47,64],"kind":[48],"has":[49],"rarely":[50],"been":[51],"published":[52],"in":[53,113],"context":[55],"widely":[58],"used":[59],"bookmark":[60,78,146],"standard\u2010setting":[61],"procedure.":[62,147],"In":[63],"article":[65],"we":[66],"examined":[67],"effect":[69],"ordered":[71,125],"item":[72,126],"booklet":[73],"difficulty":[74,95,122],"content":[76,137],"experts\u2019":[77],"judgments.":[79],"If":[80],"panelists":[81],"make":[82,140],"consistent":[84],"judgments,":[85],"their":[86,97],"resultant":[87,115],"cut":[88,116],"scores":[89,117],"should":[90],"be":[91],"unaffected":[92],"by":[93,144],"respective":[98],"booklets.":[99],"This":[100],"consistency":[102],"was":[103],"not":[104],"observed:":[105],"results":[107],"suggest":[108],"substantial":[110],"systematic":[111],"differences":[112],"can":[118],"arise":[119],"when":[120],"booklets":[127],"varies.":[128],"These":[129],"findings":[130],"raise":[131],"questions":[132],"about":[133],"ability":[135],"experts":[138],"required":[143]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":1}],"updated_date":"2026-09-21T07:43:02.447757","created_date":"2017-12-22T00:00:00"}