{"id":"https://openalex.org/W1548045340","doi":null,"title":"Reliability Testing of the AHRQ EPC Approach to Grading the Strength of Evidence in Comparative Effectiveness Reviews","display_name":"Reliability Testing of the AHRQ EPC Approach to Grading the Strength of Evidence in Comparative Effectiveness Reviews","publication_year":2012,"publication_date":"2012-04-30","ids":{"openalex":"https://openalex.org/W1548045340","mag":"1548045340"},"language":"en","primary_location":{"id":"mag:1548045340","doi":null,"is_oa":false,"landing_page_url":"https://europepmc.org/article/MED/22764383","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023729007","display_name":"Nancy D Berkman","orcid":"https://orcid.org/0000-0003-3617-1082","observed_orcids":["https://orcid.org/0000-0003-3617-1082"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nancy D Berkman","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":"https://orcid.org/0000-0003-3617-1082","affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085955825","display_name":"Kathleen N Lohr","orcid":"https://orcid.org/0000-0002-3825-0713","observed_orcids":["https://orcid.org/0000-0002-3825-0713"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kathleen N Lohr","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":"https://orcid.org/0000-0002-3825-0713","affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045563662","display_name":"Laura C Morgan","orcid":"https://orcid.org/0000-0003-1339-9746","observed_orcids":["https://orcid.org/0000-0003-1339-9746"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laura C Morgan","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":"https://orcid.org/0000-0003-1339-9746","affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053215199","display_name":"Emily Richmond","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Emily Richmond","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089058014","display_name":"Tzy\u2010Mey Kuo","orcid":"https://orcid.org/0000-0003-0700-5990","observed_orcids":["https://orcid.org/0000-0003-0700-5990"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tzy-Mey Kuo","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103695957","display_name":"Sally C. Morton","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sally C. Morton","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035597551","display_name":"Meera Viswanathan","orcid":"https://orcid.org/0000-0003-3486-8126","observed_orcids":["https://orcid.org/0000-0003-3486-8126","https://orcid.org/0000-0002-2187-2583"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Meera Viswanathan","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":"https://orcid.org/0000-0003-3486-8126","affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021169363","display_name":"Douglas B. Kamerow","orcid":"https://orcid.org/0000-0001-7000-9957","observed_orcids":["https://orcid.org/0000-0001-7000-9957"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Douglas Kamerow","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":"https://orcid.org/0000-0001-7000-9957","affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025509673","display_name":"Sue West","orcid":"https://orcid.org/0000-0003-1342-3486","observed_orcids":["https://orcid.org/0000-0003-1342-3486"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sue West","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5044626862","display_name":"Elizabeth Michelle Tant","orcid":"https://orcid.org/0000-0002-3400-6836","observed_orcids":["https://orcid.org/0000-0002-3400-6836"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elizabeth Tant","raw_affiliation_strings":["International\u2013University of North Carolina Evidence-based Practice Center"],"raw_orcid":"https://orcid.org/0000-0002-3400-6836","affiliations":[{"raw_affiliation_string":"International\u2013University of North Carolina Evidence-based Practice Center","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7884,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.76001444,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":97},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.8759999871253967,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.8759999871253967,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12443","display_name":"Delphi Technique in Research","score":0.05040000006556511,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12664","display_name":"Clinical practice guidelines implementation","score":0.03009999915957451,"subfield":{"id":"https://openalex.org/subfields/2739","display_name":"Public Health, Environmental and Occupational Health"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.9300000071525574},{"id":"https://openalex.org/keywords/risk-of-bias","display_name":"risk of bias","score":0.9409999847412109},{"id":"https://openalex.org/keywords/randomized-controlled-trials","display_name":"randomized controlled trials","score":0.8999999761581421},{"id":"https://openalex.org/keywords/observational-studies","display_name":"observational studies","score":0.9990000128746033},{"id":"https://openalex.org/keywords/reviewer-agreement","display_name":"reviewer agreement","score":0.5120000243186951},{"id":"https://openalex.org/keywords/therapeutic-interventions","display_name":"therapeutic interventions","score":0.5979999899864197},{"id":"https://openalex.org/keywords/adverse-effects","display_name":"adverse effects","score":0.5239999890327454},{"id":"https://openalex.org/keywords/outcome-domains","display_name":"outcome domains","score":0.31700000166893005}],"concepts":[{"id":"https://openalex.org/C23131810","wikidata":"https://www.wikidata.org/wiki/Q818574","display_name":"Observational study","level":2,"score":0.8153374791145325},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.6556631326675415},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.6340964436531067},{"id":"https://openalex.org/C168563851","wikidata":"https://www.wikidata.org/wiki/Q1436668","display_name":"Randomized controlled trial","level":2,"score":0.5780178904533386},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.5709575414657593},{"id":"https://openalex.org/C27415008","wikidata":"https://www.wikidata.org/wiki/Q7256382","display_name":"Psychological intervention","level":2,"score":0.5163840055465698},{"id":"https://openalex.org/C151956035","wikidata":"https://www.wikidata.org/wiki/Q1132755","display_name":"Logistic regression","level":2,"score":0.45032942295074463},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.42881259322166443},{"id":"https://openalex.org/C1862650","wikidata":"https://www.wikidata.org/wiki/Q186005","display_name":"Physical therapy","level":1,"score":0.3219503164291382},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3192453384399414},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.24035558104515076},{"id":"https://openalex.org/C159110408","wikidata":"https://www.wikidata.org/wiki/Q121176","display_name":"Nursing","level":1,"score":0.1423739194869995},{"id":"https://openalex.org/C141071460","wikidata":"https://www.wikidata.org/wiki/Q40821","display_name":"Surgery","level":1,"score":0.09630146622657776},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C138496976","wikidata":"https://www.wikidata.org/wiki/Q175002","display_name":"Developmental psychology","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:1548045340","doi":null,"is_oa":false,"landing_page_url":"https://europepmc.org/article/MED/22764383","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being","score":0.5683000087738037}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2808053597","https://openalex.org/W2017948597","https://openalex.org/W2461421262","https://openalex.org/W2004489444","https://openalex.org/W2737525774","https://openalex.org/W2056785271","https://openalex.org/W2895872414","https://openalex.org/W2061211489","https://openalex.org/W1974011311","https://openalex.org/W2888135861"],"abstract_inverted_index":{"Objectives":[0],"This":[2,553],"project":[3,34],"focused":[4,35],"on":[5,22,36,58,262,301,334,420,450,514],"Agency":[6],"for":[7,52,71,89,239,246,255,275,295,310,533,580],"Healthcare":[8],"Research":[9],"and":[10,68,76,79,96,119,128,177,197,210,251,280,317,377,443,472,485,507,606,613],"Quality":[11],"(AHRQ)":[12],"methods":[13],"guidance":[14,501,593,604],"to":[15,30,49,172,190,205,253,325,358,370,386,441,446,459,491,517,543,556,589,609,620,623],"its":[16],"Evidence-based":[17],"Practice":[18],"Center":[19],"(EPC)":[20],"program":[21],"grading":[23,50,487],"the":[24,41,46,59,87,90,131,148,156,192,198,206,218,326,340,349,413,421,519,567,603,624],"strength":[25],"of":[26,40,45,64,130,142,155,161,212,234,249,259,342,364,381,395,434,470,522,563,566,577,599,602],"evidence":[27,57,355,423,435,488,523],"(SOE)":[28],"related":[29,369],"therapeutic":[31],"interventions.":[32],"Our":[33,409,570],"inter-rater":[37,101,237],"reliability":[38,102],"testing":[39,103],"two":[42,108],"main":[43],"components":[44],"AHRQ":[47],"approach":[48],"SOE":[51,84,200,304,351,362,398,583],"specific":[53,207],"outcomes:":[54],"(1)":[55],"scoring":[56],"four":[60],"required":[61],"domains":[62,265,347],"(risk":[63],"bias,":[65],"consistency,":[66],"directness,":[67],"precision),":[69],"separately":[70],"randomized":[72],"controlled":[73],"trials":[74],"(RCTs)":[75],"observational":[77,137,256,278,382,473],"studies,":[78,360,383,474],"(2)":[80],"developing":[81],"an":[82,389],"overall":[83,350],"grade,":[85,201],"given":[86],"scores":[88,196,241,336],"individual":[91,296],"domains.":[92],"Data":[94],"Sources":[95],"Methods":[97],"We":[99,111,168,185,216,496],"conducted":[100,550],"using":[104],"data":[105],"obtained":[106],"from":[107,244],"published":[109],"CERs.":[110],"designed":[112],"10":[113,125,132],"exercises":[114,327],"(5":[115],"positive":[116],"outcomes":[117,463,476],"[benefits]":[118],"5":[120],"harms":[121],"[adverse":[122],"effects]);":[123],"all":[124,263],"included":[126,133,329],"RCTs,":[127,387],"6":[129],"1":[134],"or":[135,269,290,366,404],"more":[136,448,481,505,511],"studies.":[138],"Eleven":[140],"pairs":[141,285],"reviewers":[143,162,176,418],"(22":[144],"participants)":[145],"participated":[146],"in":[147,180,203,220,344,384,388,546,582,592],"exercises.":[149],"Each":[150],"reviewer":[151,222,227,236,284,303,540],"independently":[152],"completed":[153],"each":[154,159,182],"exercises;":[157],"subsequently,":[158],"pair":[160],"reconciled":[163,283],"their":[164,178,451,564],"independent":[165,221,235,297,302],"responses.":[166],"calculated":[169],"summary":[170],"statistics":[171],"describe":[173,191],"agreement":[174,213,238,281,314,333,338],"among":[175,214,226,282,536],"difficulty":[179,343],"making":[181],"rating":[183],"assessment.":[184],"used":[186],"logistic":[187],"regression":[188],"analysis":[189],"relationship":[193],"between":[194],"domain":[195,240,311,335,372],"final":[199],"both":[202],"relation":[204],"grade":[208,399],"selected":[209],"level":[211,233,341],"reviewers.":[215,298,538],"examined":[217],"change":[219],"ratings":[223],"following":[224],"reconciliation":[225],"pairs.":[228],"Results":[230],"The":[232,379],"varied":[242],"considerably":[243],"substantial":[245],"RCT":[247,359,371],"risk":[248,258],"bias":[250],"directness":[252],"slight":[254,316],"study":[257,571],"bias.":[260],"Agreement":[261,271,300],"other":[264],"was":[266,272,286,294,306,315,319,356,391],"either":[267,402],"moderate":[268,365],"fair.":[270],"generally":[273,307],"better":[274,291,322,361],"RCTs":[276,471],"than":[277,292,309,482],"studies":[279],"as":[287,289,524],"good":[288],"it":[293,318],"grades":[305,363],"poorer":[308,397],"scores.":[312],"Overall":[313],"not":[320,438],"appreciably":[321],"when":[323,428],"limited":[324,357],"that":[328,412,436,464,477,489,498,558,594,611,616],"only":[330,573],"RCTs.":[331],"Neither":[332],"nor":[337],"about":[339],"evaluating":[345],"particular":[346,455],"predicted":[348],"grades.":[352],"When":[354],"high":[367],"were":[368],"scores\u2019":[373],"being":[374],"considered":[375],"consistent":[376],"precise.":[378],"inclusion":[380],"addition":[385],"exercise":[390],"a":[392,396,468,574],"strong":[393],"predictor":[394],"\u2014":[400],"namely,":[401],"insufficient":[403,600],"low.":[405],"Conclusions":[407],"findings":[410],"demonstrate":[411],"conclusions":[414],"reached":[415],"by":[416,510],"experienced":[417,537],"based":[419],"same":[422],"can":[424,560],"differ":[425],"greatly,":[426],"particularly":[427,513],"they":[429,444,548],"are":[430,465,478],"faced":[431],"with":[432,461],"bodies":[433,521],"do":[437],"lend":[439],"themselves":[440],"meta-analysis":[442],"need":[445,542,619],"rely":[447],"heavily":[449],"own":[452],"judgment.":[453],"Of":[454],"concern":[456],"is":[457,502,587],"how":[458,515,547,607],"deal":[460],"(a)":[462],"evaluated":[466,479],"through":[467,480],"combination":[469],"(b)":[475],"one":[483],"measure":[484],"(c)":[486],"appears":[490],"show":[492],"no":[493],"difference.":[494],"conclude":[497],"additional":[499],"methodological":[500],"needed,":[503],"including":[504],"details":[506],"examples,":[508],"supported":[509],"training,":[512],"best":[516,608],"evaluate":[518],"\u201cthornier\u201d":[520],"discussed":[525],"above.":[526],"However,":[527],"some":[528],"potential":[529],"will":[530,554],"always":[531],"exist":[532],"disagreement":[534],"even":[535],"EPC":[539],"teams":[541],"be":[544,561,596,621],"transparent":[545],"have":[549],"this":[551],"task.":[552],"help":[555],"ensure":[557],"stakeholders":[559],"confident":[562],"interpretation":[565],"evidence.":[568],"provided":[572],"first":[575],"approximation":[576],"reviewers\u2019":[578],"rationales":[579],"differences":[581],"decisions.":[584],"Additional":[585],"research":[586],"needed":[588],"understand":[590],"gaps":[591],"should":[595],"filled,":[597],"areas":[598],"understanding":[601],"itself":[605],"overcome":[610],"deficit,":[612],"complex":[614],"decisions":[615],"may":[617],"still":[618],"left":[622],"review":[625],"team\u2019s":[626],"substantive":[627],"expertise.":[628]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2018,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":1}],"updated_date":"2026-10-17T09:40:24.721904","created_date":"2016-06-24T00:00:00"}