{"id":"https://openalex.org/W2308382125","doi":"https://doi.org/10.1117/12.2217074","title":"MRMC analysis of agreement studies","display_name":"MRMC analysis of agreement studies","publication_year":2016,"publication_date":"2016-03-24","ids":{"openalex":"https://openalex.org/W2308382125","doi":"https://doi.org/10.1117/12.2217074","mag":"2308382125","pmid":"https://pubmed.ncbi.nlm.nih.gov/28794577"},"language":"en","primary_location":{"id":"doi:10.1117/12.2217074","is_oa":false,"landing_page_url":"https://doi.org/10.1117/12.2217074","pdf_url":null,"source":{"id":"https://openalex.org/S183492911","display_name":"Proceedings of SPIE, the International Society for Optical Engineering/Proceedings of SPIE","issn_l":"0277-786X","issn":["0277-786X","1996-756X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315543","host_organization_name":"SPIE","host_organization_lineage":["https://openalex.org/P4310315543"],"host_organization_lineage_names":["SPIE"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SPIE Proceedings","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/5546377","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067893440","display_name":"Brandon D. Gallas","orcid":"https://orcid.org/0000-0001-7332-1620"},"institutions":[{"id":"https://openalex.org/I1320320070","display_name":"United States Food and Drug Administration","ror":"https://ror.org/034xvzb47","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1320320070"]},{"id":"https://openalex.org/I4210119464","display_name":"Quality and Reliability (Greece)","ror":"https://ror.org/02f8mda22","country_code":"GR","type":"company","lineage":["https://openalex.org/I4210119464"]}],"countries":["GR","US"],"is_corresponding":false,"raw_author_name":"Brandon D. Gallas","raw_affiliation_strings":["CDRH/OSEL Division of Imaging, Diagnostics, and Software Reliability, 10903 New Hampshire Ave, Silver Spring, MD, 20993","U.S. Food and Drug Administration (United States)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CDRH/OSEL Division of Imaging, Diagnostics, and Software Reliability, 10903 New Hampshire Ave, Silver Spring, MD, 20993","institution_ids":["https://openalex.org/I4210119464"]},{"raw_affiliation_string":"U.S. Food and Drug Administration (United States)","institution_ids":["https://openalex.org/I1320320070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024182586","display_name":"Amrita Anam","orcid":null},"institutions":[{"id":"https://openalex.org/I1320320070","display_name":"United States Food and Drug Administration","ror":"https://ror.org/034xvzb47","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1320320070"]},{"id":"https://openalex.org/I4210119464","display_name":"Quality and Reliability (Greece)","ror":"https://ror.org/02f8mda22","country_code":"GR","type":"company","lineage":["https://openalex.org/I4210119464"]},{"id":"https://openalex.org/I79272384","display_name":"University of Maryland, Baltimore County","ror":"https://ror.org/02qskvh78","country_code":"US","type":"education","lineage":["https://openalex.org/I79272384"]}],"countries":["GR","US"],"is_corresponding":false,"raw_author_name":"Amrita Anam","raw_affiliation_strings":["CDRH/OSEL Division of Imaging, Diagnostics, and Software Reliability, 10903 New Hampshire Ave, Silver Spring, MD, 20993","UMBC, Department of Information Systems, 1000 Hilltop Cir, Baltimore, MD 21250","U.S. Food and Drug Administration (United States)","Univ. of Maryland, Baltimore County (United States)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CDRH/OSEL Division of Imaging, Diagnostics, and Software Reliability, 10903 New Hampshire Ave, Silver Spring, MD, 20993","institution_ids":["https://openalex.org/I4210119464"]},{"raw_affiliation_string":"UMBC, Department of Information Systems, 1000 Hilltop Cir, Baltimore, MD 21250","institution_ids":["https://openalex.org/I79272384"]},{"raw_affiliation_string":"U.S. Food and Drug Administration (United States)","institution_ids":["https://openalex.org/I1320320070"]},{"raw_affiliation_string":"Univ. of Maryland, Baltimore County (United States)","institution_ids":["https://openalex.org/I79272384"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100686046","display_name":"Weijie Chen","orcid":"https://orcid.org/0000-0001-6601-2483"},"institutions":[{"id":"https://openalex.org/I1320320070","display_name":"United States Food and Drug Administration","ror":"https://ror.org/034xvzb47","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1320320070"]},{"id":"https://openalex.org/I4210119464","display_name":"Quality and Reliability (Greece)","ror":"https://ror.org/02f8mda22","country_code":"GR","type":"company","lineage":["https://openalex.org/I4210119464"]}],"countries":["GR","US"],"is_corresponding":false,"raw_author_name":"Weijie Chen","raw_affiliation_strings":["CDRH/OSEL Division of Imaging, Diagnostics, and Software Reliability, 10903 New Hampshire Ave, Silver Spring, MD, 20993","U.S. Food and Drug Administration  (United States)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CDRH/OSEL Division of Imaging, Diagnostics, and Software Reliability, 10903 New Hampshire Ave, Silver Spring, MD, 20993","institution_ids":["https://openalex.org/I4210119464"]},{"raw_affiliation_string":"U.S. Food and Drug Administration  (United States)","institution_ids":["https://openalex.org/I1320320070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017171236","display_name":"Adam Wunderlich","orcid":"https://orcid.org/0000-0002-8463-9156"},"institutions":[{"id":"https://openalex.org/I1320320070","display_name":"United States Food and Drug Administration","ror":"https://ror.org/034xvzb47","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1320320070"]},{"id":"https://openalex.org/I4210119464","display_name":"Quality and Reliability (Greece)","ror":"https://ror.org/02f8mda22","country_code":"GR","type":"company","lineage":["https://openalex.org/I4210119464"]}],"countries":["GR","US"],"is_corresponding":false,"raw_author_name":"Adam Wunderlich","raw_affiliation_strings":["CDRH/OSEL Division of Imaging, Diagnostics, and Software Reliability, 10903 New Hampshire Ave, Silver Spring, MD, 20993","U.S. Food and Drug Administration (United States)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CDRH/OSEL Division of Imaging, Diagnostics, and Software Reliability, 10903 New Hampshire Ave, Silver Spring, MD, 20993","institution_ids":["https://openalex.org/I4210119464"]},{"raw_affiliation_string":"U.S. Food and Drug Administration (United States)","institution_ids":["https://openalex.org/I1320320070"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100347599","display_name":"Zhiwei Zhang","orcid":"https://orcid.org/0000-0002-3187-9180"},"institutions":[{"id":"https://openalex.org/I1320320070","display_name":"United States Food and Drug Administration","ror":"https://ror.org/034xvzb47","country_code":"US","type":"government","lineage":["https://openalex.org/I1299022934","https://openalex.org/I1320320070"]},{"id":"https://openalex.org/I4210134973","display_name":"MRC Biostatistics Unit","ror":"https://ror.org/046vje122","country_code":"GB","type":"facility","lineage":["https://openalex.org/I241749","https://openalex.org/I4210087105","https://openalex.org/I4210134973","https://openalex.org/I90344618"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Zhiwei Zhang","raw_affiliation_strings":["CDRH/OSB Division of Biostatistics, 10903 New Hampshire Ave, Silver Spring, MD, 20993","U.S. Food and Drug Administration (United States)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CDRH/OSB Division of Biostatistics, 10903 New Hampshire Ave, Silver Spring, MD, 20993","institution_ids":["https://openalex.org/I4210134973"]},{"raw_affiliation_string":"U.S. Food and Drug Administration (United States)","institution_ids":["https://openalex.org/I1320320070"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.01728583,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":95},"biblio":{"volume":"9787","issue":null,"first_page":"97870F","last_page":"97870F"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11700","display_name":"Hemodynamic Monitoring and Therapy","score":0.9891999959945679,"subfield":{"id":"https://openalex.org/subfields/2746","display_name":"Surgery"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11235","display_name":"Statistical Methods in Clinical Trials","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/statistic","display_name":"Statistic","score":0.7241964340209961},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6408002376556396},{"id":"https://openalex.org/keywords/concordance","display_name":"Concordance","score":0.640643298625946},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.6010262966156006},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.5555110573768616},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.502479076385498},{"id":"https://openalex.org/keywords/property","display_name":"Property (philosophy)","score":0.4511893093585968},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3289659023284912},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.32284802198410034},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.26959502696990967},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.12652063369750977}],"concepts":[{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.7241964340209961},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6408002376556396},{"id":"https://openalex.org/C160798450","wikidata":"https://www.wikidata.org/wiki/Q4230870","display_name":"Concordance","level":2,"score":0.640643298625946},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.6010262966156006},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.5555110573768616},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.502479076385498},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.4511893093585968},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3289659023284912},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.32284802198410034},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26959502696990967},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.12652063369750977},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1117/12.2217074","is_oa":false,"landing_page_url":"https://doi.org/10.1117/12.2217074","pdf_url":null,"source":{"id":"https://openalex.org/S183492911","display_name":"Proceedings of SPIE, the International Society for Optical Engineering/Proceedings of SPIE","issn_l":"0277-786X","issn":["0277-786X","1996-756X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315543","host_organization_name":"SPIE","host_organization_lineage":["https://openalex.org/P4310315543"],"host_organization_lineage_names":["SPIE"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SPIE Proceedings","raw_type":"proceedings-article"},{"id":"pmid:28794577","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/28794577","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of SPIE--the International Society for Optical Engineering","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:5546377","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/5546377","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc SPIE Int Soc Opt Eng","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:5546377","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/5546377","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc SPIE Int Soc Opt Eng","raw_type":"Text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W1972019717","https://openalex.org/W1994633745","https://openalex.org/W2030346400","https://openalex.org/W2073234747","https://openalex.org/W2085205305","https://openalex.org/W2087189170","https://openalex.org/W2124379907","https://openalex.org/W4249999102","https://openalex.org/W4293257135","https://openalex.org/W6684539355"],"related_works":["https://openalex.org/W1601684588","https://openalex.org/W2108427450","https://openalex.org/W2104328188","https://openalex.org/W648938440","https://openalex.org/W4220795960","https://openalex.org/W2789177613","https://openalex.org/W2396330467","https://openalex.org/W2915108716","https://openalex.org/W1517024399","https://openalex.org/W2506500726"],"abstract_inverted_index":{"The":[0],"purpose":[1],"of":[2,82,104,158],"this":[3,189],"work":[4],"is":[5,115,143,150,161,164,174,180,185],"to":[6,14,28,110],"present":[7],"and":[8,37,47,77,135,147,191,208,216],"evaluate":[9,57,69,79],"methods":[10,27],"based":[11],"on":[12,61,72,85],"U-statistics":[13],"compare":[15],"intra-":[16],"or":[17,177],"inter-reader":[18],"agreement":[19,36,100,112],"across":[20],"different":[21],"imaging":[22],"modalities.":[23],"We":[24,33],"apply":[25,109],"these":[26],"multi-reader":[29],"multi-case":[30],"(MRMC)":[31],"studies.":[32],"measure":[34],"reader-averaged":[35],"estimate":[38],"its":[39,148,167],"variance":[40,149,215],"accounting":[41],"for":[42,200],"the":[43,70,83,92,99,105,116,125,183,186,197],"variability":[44],"from":[45],"readers":[46,120],"cases":[48,123],"(an":[49],"MRMC":[50],"analysis).":[51],"In":[52,91],"our":[53,214],"application,":[54],"pathologists":[55],"(readers)":[56],"patient":[58],"tissue":[59],"mounted":[60],"glass":[62],"slides":[63,71,84],"(cases)":[64],"in":[65,124,166],"two":[66,119,122],"ways.":[67],"They":[68],"a":[73,86,133,159,192,204],"microscope":[74],"(reference":[75],"modality)":[76],"they":[78],"digital":[80],"scans":[81],"computer":[87],"display":[88],"(new":[89],"modality).":[90],"current":[93],"work,":[94],"we":[95,202],"consider":[96],"concordance":[97],"as":[98],"measure,":[101],"but":[102],"many":[103],"concepts":[106],"outlined":[107],"here":[108],"other":[111],"measures.":[113],"Concordance":[114,128],"probability":[117],"that":[118,162,213],"rank":[121],"same":[126],"order.":[127],"can":[129],"be":[130],"estimated":[131],"with":[132],"U-statistic":[134,160,198],"thus":[136],"it":[137,142,163,169],"has":[138],"some":[139],"nice":[140],"properties:":[141],"unbiased,":[144],"asymptotically":[145],"normal,":[146],"given":[151],"by":[152],"an":[153],"explicit":[154],"formula.":[155],"Another":[156],"property":[157,190],"symmetric":[165],"inputs;":[168],"doesn't":[170],"matter":[171],"which":[172,178],"reader":[173],"listed":[175,181],"first":[176],"case":[179],"first,":[182],"result":[184],"same.":[187],"Using":[188],"few":[193],"tricks":[194],"while":[195],"building":[196],"kernel":[199],"concordance,":[201],"get":[203],"mathematically":[205],"tractable":[206],"problem":[207],"efficient":[209],"software.":[210],"Simulations":[211],"show":[212],"covariance":[217],"estimates":[218],"are":[219],"unbiased.":[220]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}