{"id":"https://openalex.org/W2739815944","doi":"https://doi.org/10.1044/2017_aja-16-0100","title":"Evaluating Random Error in Clinician-Administered Surveys: Theoretical Considerations and Clinical Applications of Interobserver Reliability and Agreement","display_name":"Evaluating Random Error in Clinician-Administered Surveys: Theoretical Considerations and Clinical Applications of Interobserver Reliability and Agreement","publication_year":2017,"publication_date":"2017-08-02","ids":{"openalex":"https://openalex.org/W2739815944","doi":"https://doi.org/10.1044/2017_aja-16-0100","mag":"2739815944","pmid":"https://pubmed.ncbi.nlm.nih.gov/28768319"},"language":"en","primary_location":{"id":"doi:10.1044/2017_aja-16-0100","doi":"https://doi.org/10.1044/2017_aja-16-0100","is_oa":false,"landing_page_url":"https://doi.org/10.1044/2017_aja-16-0100","pdf_url":null,"source":{"id":"https://openalex.org/S37662143","display_name":"American Journal of Audiology","issn_l":"1059-0889","issn":["1059-0889","1558-9137"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320281","host_organization_name":"American Speech\u2013Language\u2013Hearing Association","host_organization_lineage":["https://openalex.org/P4310320281"],"host_organization_lineage_names":["American Speech\u2013Language\u2013Hearing Association"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American Journal of Audiology","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5038120297","display_name":"Rebecca J. Bennett","orcid":"https://orcid.org/0000-0001-9427-5539","observed_orcids":["https://orcid.org/0000-0001-9427-5539"]},"institutions":[{"id":"https://openalex.org/I177877127","display_name":"The University of Western Australia","ror":"https://ror.org/047272k79","country_code":"AU","type":"education","lineage":["https://openalex.org/I177877127"]},{"id":"https://openalex.org/I4210164047","display_name":"Ear Science Institute Australia","ror":"https://ror.org/05f5rab97","country_code":"AU","type":"nonprofit","lineage":["https://openalex.org/I4210164047"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Rebecca J. Bennett","raw_affiliation_strings":["Ear Science Institute Australia, Subiaco, Western Australia","Ear Sciences Centre, The University of Western Australia, Nedlands","Google Scholar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ear Science Institute Australia, Subiaco, Western Australia","institution_ids":["https://openalex.org/I4210164047"]},{"raw_affiliation_string":"Ear Sciences Centre, The University of Western Australia, Nedlands","institution_ids":["https://openalex.org/I177877127"]},{"raw_affiliation_string":"Google Scholar","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072779082","display_name":"Dunay Schmulian","orcid":"https://orcid.org/0000-0003-1203-4032","observed_orcids":["https://orcid.org/0000-0003-1203-4032"]},"institutions":[{"id":"https://openalex.org/I177877127","display_name":"The University of Western Australia","ror":"https://ror.org/047272k79","country_code":"AU","type":"education","lineage":["https://openalex.org/I177877127"]},{"id":"https://openalex.org/I2799506148","display_name":"Government of Western Australia","ror":"https://ror.org/00wqdbc63","country_code":"AU","type":"government","lineage":["https://openalex.org/I2799506148"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Dunay S. Taljaard","raw_affiliation_strings":["Ear Sciences Centre, The University of Western Australia, Nedlands","Princess Margaret Hospital, Subiaco, Western Australia","Google Scholar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ear Sciences Centre, The University of Western Australia, Nedlands","institution_ids":["https://openalex.org/I177877127"]},{"raw_affiliation_string":"Princess Margaret Hospital, Subiaco, Western Australia","institution_ids":["https://openalex.org/I2799506148"]},{"raw_affiliation_string":"Google Scholar","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077662386","display_name":"Michelle Olaithe","orcid":"https://orcid.org/0000-0002-9539-5716","observed_orcids":["https://orcid.org/0000-0002-9539-5716"]},"institutions":[{"id":"https://openalex.org/I177877127","display_name":"The University of Western Australia","ror":"https://ror.org/047272k79","country_code":"AU","type":"education","lineage":["https://openalex.org/I177877127"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Michelle Olaithe","raw_affiliation_strings":["School of Psychological Science, The University of Western Australia","Google Scholar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Psychological Science, The University of Western Australia","institution_ids":["https://openalex.org/I177877127"]},{"raw_affiliation_string":"Google Scholar","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025245610","display_name":"Christopher G. Brennan\u2010Jones","orcid":"https://orcid.org/0000-0001-7216-8097","observed_orcids":["https://orcid.org/0000-0001-7216-8097"]},"institutions":[{"id":"https://openalex.org/I177877127","display_name":"The University of Western Australia","ror":"https://ror.org/047272k79","country_code":"AU","type":"education","lineage":["https://openalex.org/I177877127"]},{"id":"https://openalex.org/I4210164047","display_name":"Ear Science Institute Australia","ror":"https://ror.org/05f5rab97","country_code":"AU","type":"nonprofit","lineage":["https://openalex.org/I4210164047"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Chris Brennan-Jones","raw_affiliation_strings":["Ear Science Institute Australia, Subiaco, Western Australia","Ear Sciences Centre, The University of Western Australia, Nedlands","Google Scholar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ear Science Institute Australia, Subiaco, Western Australia","institution_ids":["https://openalex.org/I4210164047"]},{"raw_affiliation_string":"Ear Sciences Centre, The University of Western Australia, Nedlands","institution_ids":["https://openalex.org/I177877127"]},{"raw_affiliation_string":"Google Scholar","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040605783","display_name":"Robert Henry Eikelboom","orcid":"https://orcid.org/0000-0003-2911-5381","observed_orcids":["https://orcid.org/0000-0003-2911-5381"]},"institutions":[{"id":"https://openalex.org/I177877127","display_name":"The University of Western Australia","ror":"https://ror.org/047272k79","country_code":"AU","type":"education","lineage":["https://openalex.org/I177877127"]},{"id":"https://openalex.org/I4210164047","display_name":"Ear Science Institute Australia","ror":"https://ror.org/05f5rab97","country_code":"AU","type":"nonprofit","lineage":["https://openalex.org/I4210164047"]},{"id":"https://openalex.org/I69552723","display_name":"University of Pretoria","ror":"https://ror.org/00g0p6g84","country_code":"ZA","type":"education","lineage":["https://openalex.org/I69552723"]}],"countries":["AU","ZA"],"is_corresponding":false,"raw_author_name":"Robert H. Eikelboom","raw_affiliation_strings":["Department of Speech-Language Pathology and Audiology, University of Pretoria, South Africa","Ear Science Institute Australia, Subiaco, Western Australia","Ear Sciences Centre, The University of Western Australia, Nedlands","Google Scholar"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Speech-Language Pathology and Audiology, University of Pretoria, South Africa","institution_ids":["https://openalex.org/I69552723"]},{"raw_affiliation_string":"Ear Science Institute Australia, Subiaco, Western Australia","institution_ids":["https://openalex.org/I4210164047"]},{"raw_affiliation_string":"Ear Sciences Centre, The University of Western Australia, Nedlands","institution_ids":["https://openalex.org/I177877127"]},{"raw_affiliation_string":"Google Scholar","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8198,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.76466754,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":97},"biblio":{"volume":"26","issue":"3","first_page":"191","last_page":"201"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10283","display_name":"Hearing Loss and Rehabilitation","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/3616","display_name":"Speech and Hearing"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11539","display_name":"Survey Methodology and Nonresponse","score":0.0026000000070780516,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"inter-rater reliability","score":0.9769999980926514},{"id":"https://openalex.org/keywords/interobserver-agreement","display_name":"interobserver agreement","score":0.9700000286102295},{"id":"https://openalex.org/keywords/random-error","display_name":"random error","score":0.640999972820282},{"id":"https://openalex.org/keywords/cohen-s-kappa","display_name":"Cohen's kappa","score":0.6809999942779541},{"id":"https://openalex.org/keywords/krippendorff-s-alpha","display_name":"Krippendorff's alpha","score":1.0},{"id":"https://openalex.org/keywords/interclass-correlation","display_name":"interclass correlation","score":0.9900000095367432},{"id":"https://openalex.org/keywords/data-types","display_name":"data types","score":0.47200000286102295}],"concepts":[{"id":"https://openalex.org/C160798450","wikidata":"https://www.wikidata.org/wiki/Q4230870","display_name":"Concordance","level":2,"score":0.9070008397102356},{"id":"https://openalex.org/C2778724333","wikidata":"https://www.wikidata.org/wiki/Q14401","display_name":"Kappa","level":2,"score":0.7048119306564331},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.6998398900032043},{"id":"https://openalex.org/C104709138","wikidata":"https://www.wikidata.org/wiki/Q1671540","display_name":"Intraclass correlation","level":3,"score":0.671748161315918},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6555052995681763},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.6459550261497498},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.5722452402114868},{"id":"https://openalex.org/C163864269","wikidata":"https://www.wikidata.org/wiki/Q1107106","display_name":"Cohen's kappa","level":2,"score":0.5394049882888794},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.4719395041465759},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.425799697637558},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4171745777130127},{"id":"https://openalex.org/C69941244","wikidata":"https://www.wikidata.org/wiki/Q6045770","display_name":"Interclass correlation","level":4,"score":0.410234272480011},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.33201655745506287},{"id":"https://openalex.org/C70410870","wikidata":"https://www.wikidata.org/wiki/Q199906","display_name":"Clinical psychology","level":1,"score":0.27335745096206665},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.24977505207061768},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.21311303973197937},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.11755865812301636},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D000072018","descriptor_name":"Audiologists","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000328","descriptor_name":"Adult","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000368","descriptor_name":"Aged","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000369","descriptor_name":"Aged, 80 and over","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D005260","descriptor_name":"Female","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006310","descriptor_name":"Hearing Aids","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008297","descriptor_name":"Male","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D008875","descriptor_name":"Middle Aged","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011446","descriptor_name":"Prospective Studies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D011795","descriptor_name":"Surveys and Questionnaires","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012648","descriptor_name":"Self Care","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D015203","descriptor_name":"Reproducibility of Results","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015331","descriptor_name":"Cohort Studies","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D015588","descriptor_name":"Observer Variation","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D034381","descriptor_name":"Hearing Loss","qualifier_ui":"Q000534","qualifier_name":"rehabilitation","is_major_topic":true}],"locations_count":3,"locations":[{"id":"doi:10.1044/2017_aja-16-0100","doi":"https://doi.org/10.1044/2017_aja-16-0100","is_oa":false,"landing_page_url":"https://doi.org/10.1044/2017_aja-16-0100","pdf_url":null,"source":{"id":"https://openalex.org/S37662143","display_name":"American Journal of Audiology","issn_l":"1059-0889","issn":["1059-0889","1558-9137"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","dongbi-c","doyens","jufo-1","ki-jl-1","medline","norway-1","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320281","host_organization_name":"American Speech\u2013Language\u2013Hearing Association","host_organization_lineage":["https://openalex.org/P4310320281"],"host_organization_lineage_names":["American Speech\u2013Language\u2013Hearing Association"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American Journal of Audiology","raw_type":"journal-article"},{"id":"pmid:28768319","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/28768319","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"American journal of audiology","raw_type":"Journal Article"},{"id":"pmh:oai:pure.atira.dk:publications/e0ca3006-2d3d-4085-a7f0-0444e69f495a","doi":null,"is_oa":false,"landing_page_url":"https://research-repository.uwa.edu.au/en/publications/e0ca3006-2d3d-4085-a7f0-0444e69f495a","pdf_url":null,"source":{"id":"https://openalex.org/S4306402492","display_name":"UWA Profiles and Research Repository (UWA)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I177877127","host_organization_name":"The University of Western Australia","host_organization_lineage":["https://openalex.org/I177877127"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Bennett, R J, Taljaard, D S, Olaithe, M, Brennan-Jones, C & Eikelboom, R H 2017, 'Evaluating random error in clinician-administered surveys : Theoretical considerations and clinical applications of interobserver reliability and agreement', American Journal of Audiology, vol. 26, no. 3, pp. 191-201. https://doi.org/10.1044/2017_AJA-16-0100","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5683000087738037,"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being"}],"study_designs":[{"id":"https://openalex.org/study-designs/observational-study","display_name":"Observational Study"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W1500217743","https://openalex.org/W1562303665","https://openalex.org/W1756911457","https://openalex.org/W1975879668","https://openalex.org/W1996700455","https://openalex.org/W2021188434","https://openalex.org/W2026096417","https://openalex.org/W2053154970","https://openalex.org/W2059642721","https://openalex.org/W2061504941","https://openalex.org/W2068969116","https://openalex.org/W2069378996","https://openalex.org/W2076870549","https://openalex.org/W2081248103","https://openalex.org/W2081791869","https://openalex.org/W2085552013","https://openalex.org/W2086059573","https://openalex.org/W2087560253","https://openalex.org/W2090280719","https://openalex.org/W2091019309","https://openalex.org/W2102380164","https://openalex.org/W2104741198","https://openalex.org/W2105374537","https://openalex.org/W2115988342","https://openalex.org/W2131915193","https://openalex.org/W2143351518","https://openalex.org/W2155243985","https://openalex.org/W2156813589","https://openalex.org/W2164777277","https://openalex.org/W2170021941","https://openalex.org/W2220219025","https://openalex.org/W2495099234","https://openalex.org/W2582743722","https://openalex.org/W2588050207","https://openalex.org/W3141544461"],"related_works":["https://openalex.org/W2791854229","https://openalex.org/W4233276225","https://openalex.org/W1500217743","https://openalex.org/W2164777277","https://openalex.org/W2153093793","https://openalex.org/W2914407108","https://openalex.org/W1965000394","https://openalex.org/W4405654390","https://openalex.org/W3205630302","https://openalex.org/W2004406411"],"abstract_inverted_index":{"PURPOSE:":[0],"The":[1,101],"purpose":[2],"of":[3,10,25,36,84,94,103,107,118],"this":[4,59],"study":[5,140],"is":[6],"to":[7,31,69,80,126,146],"raise":[8],"awareness":[9],"interobserver":[11,17,119],"concordance":[12,88,108,120,150],"and":[13,19,44,96,134,157],"the":[14,23,33,74,116,143,153],"differences":[15],"between":[16],"reliability":[18,95],"agreement":[20,97],"when":[21],"evaluating":[22],"responsiveness":[24],"a":[26,92,112],"clinician-administered":[27],"survey":[28,148],"and,":[29],"specifically,":[30],"demonstrate":[32],"clinical":[34,64,160],"implications":[35],"data":[37,132],"types":[38,133],"(nominal/categorical,":[39],"ordinal,":[40],"interval,":[41],"or":[42,54],"ratio)":[43],"statistical":[45,98,105,135,154],"index":[46,136],"selection":[47],"(for":[48],"example,":[49,114],"Cohen's":[50],"kappa,":[51],"Krippendorff's":[52],"alpha,":[53],"interclass":[55],"correlation).":[56],"METHODS:":[57],"In":[58],"prospective":[60],"cohort":[61],"study,":[62],"3":[63],"audiologists,":[65],"who":[66],"were":[67],"masked":[68],"each":[70],"other's":[71],"scores,":[72],"administered":[73],"Practical":[75],"Hearing":[76],"Aid":[77],"Skills":[78],"Test-Revised":[79],"18":[81],"adult":[82],"owners":[83],"hearing":[85],"aids.":[86],"Interobserver":[87],"was":[89,109],"examined":[90],"using":[91],"range":[93],"indices.":[99],"RESULTS:":[100],"importance":[102],"selecting":[104],"measures":[106],"demonstrated":[110],"with":[111],"worked":[113],"wherein":[115],"level":[117],"achieved":[121],"varied":[122],"from":[123],"\"no":[124],"agreement\"":[125,129],"\"almost":[127],"perfect":[128],"depending":[130],"on":[131],"selected.":[137],"CONCLUSIONS:":[138],"This":[139],"demonstrates":[141],"that":[142],"methodology":[144],"used":[145],"evaluate":[147],"score":[149],"can":[151],"influence":[152],"results":[155],"obtained":[156],"thus":[158],"affect":[159],"interpretations.":[161]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2017-08-08T00:00:00"}