{"id":"https://openalex.org/W1559121794","doi":"https://doi.org/10.1109/tip.2016.2544703","title":"An Empirical Study Into Annotator Agreement, Ground Truth Estimation, and Algorithm Evaluation","display_name":"An Empirical Study Into Annotator Agreement, Ground Truth Estimation, and Algorithm Evaluation","publication_year":2016,"publication_date":"2016-03-21","ids":{"openalex":"https://openalex.org/W1559121794","doi":"https://doi.org/10.1109/tip.2016.2544703","mag":"1559121794","pmid":"https://pubmed.ncbi.nlm.nih.gov/27019487"},"language":"en","primary_location":{"id":"doi:10.1109/tip.2016.2544703","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tip.2016.2544703","pdf_url":null,"source":{"id":"https://openalex.org/S4210173141","display_name":"IEEE Transactions on Image Processing","issn_l":"1057-7149","issn":["1057-7149","1941-0042"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Image Processing","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1307.0426","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016714459","display_name":"Thomas Lampert","orcid":"https://orcid.org/0000-0002-4911-3941"},"institutions":[{"id":"https://openalex.org/I4210100283","display_name":"Laboratoire des Sciences de l'Ing\u00e9nieur, de l'Informatique et de l'Imagerie","ror":"https://ror.org/00k4e5n71","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I2801509770","https://openalex.org/I4210095849","https://openalex.org/I4210100283","https://openalex.org/I4210159245","https://openalex.org/I68947357"]},{"id":"https://openalex.org/I68947357","display_name":"Universit\u00e9 de Strasbourg","ror":"https://ror.org/00pg6eq24","country_code":"FR","type":"education","lineage":["https://openalex.org/I68947357"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Thomas A. Lampert","raw_affiliation_strings":["ICube Laboratory, University of Strasbourg, Strasbourg, France","Laboratoire des sciences de l'ing\u00e9nieur, de l'informatique et de l'imagerie"],"affiliations":[{"raw_affiliation_string":"ICube Laboratory, University of Strasbourg, Strasbourg, France","institution_ids":["https://openalex.org/I68947357"]},{"raw_affiliation_string":"Laboratoire des sciences de l'ing\u00e9nieur, de l'informatique et de l'imagerie","institution_ids":["https://openalex.org/I4210100283"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112826116","display_name":"Andr\u00e9 Stumpf","orcid":null},"institutions":[{"id":"https://openalex.org/I148261361","display_name":"Institut de physique du globe de Paris","ror":"https://ror.org/004gzqz66","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I148261361","https://openalex.org/I204730241","https://openalex.org/I4210148025"]},{"id":"https://openalex.org/I4210147109","display_name":"Laboratoire Image, Ville, Environnement","ror":"https://ror.org/03x8fem72","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I4210107625","https://openalex.org/I4210147109","https://openalex.org/I68947357"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Andre Stumpf","raw_affiliation_strings":["Laboratoire Image, Ville, et Environment (LIVE), University of Strasbourg, Strasbourg, France","Institut de physique du globe de Strasbourg"],"affiliations":[{"raw_affiliation_string":"Laboratoire Image, Ville, et Environment (LIVE), University of Strasbourg, Strasbourg, France","institution_ids":["https://openalex.org/I4210147109"]},{"raw_affiliation_string":"Institut de physique du globe de Strasbourg","institution_ids":["https://openalex.org/I148261361"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031872978","display_name":"Pierre Gan\u00e7arski","orcid":"https://orcid.org/0000-0003-1230-6560"},"institutions":[{"id":"https://openalex.org/I4210100283","display_name":"Laboratoire des Sciences de l'Ing\u00e9nieur, de l'Informatique et de l'Imagerie","ror":"https://ror.org/00k4e5n71","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I2801509770","https://openalex.org/I4210095849","https://openalex.org/I4210100283","https://openalex.org/I4210159245","https://openalex.org/I68947357"]},{"id":"https://openalex.org/I68947357","display_name":"Universit\u00e9 de Strasbourg","ror":"https://ror.org/00pg6eq24","country_code":"FR","type":"education","lineage":["https://openalex.org/I68947357"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Pierre Gancarski","raw_affiliation_strings":["ICube Laboratory, University of Strasbourg, Strasbourg, France","Laboratoire des sciences de l'ing\u00e9nieur, de l'informatique et de l'imagerie"],"affiliations":[{"raw_affiliation_string":"ICube Laboratory, University of Strasbourg, Strasbourg, France","institution_ids":["https://openalex.org/I68947357"]},{"raw_affiliation_string":"Laboratoire des sciences de l'ing\u00e9nieur, de l'informatique et de l'imagerie","institution_ids":["https://openalex.org/I4210100283"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5016714459"],"corresponding_institution_ids":["https://openalex.org/I4210100283","https://openalex.org/I68947357"],"apc_list":null,"apc_paid":null,"fwci":11.9359,"has_fulltext":false,"cited_by_count":92,"citation_normalized_percentile":{"value":0.98044741,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"25","issue":"6","first_page":"2557","last_page":"2572"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9176999926567078,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9176999926567078,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.009499999694526196,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.007400000002235174,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.7055281400680542},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5996735692024231},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5723658800125122},{"id":"https://openalex.org/keywords/estimation","display_name":"Estimation","score":0.4906977415084839},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38069531321525574},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08160173892974854}],"concepts":[{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.7055281400680542},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5996735692024231},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5723658800125122},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.4906977415084839},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38069531321525574},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08160173892974854},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tip.2016.2544703","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tip.2016.2544703","pdf_url":null,"source":{"id":"https://openalex.org/S4210173141","display_name":"IEEE Transactions on Image Processing","issn_l":"1057-7149","issn":["1057-7149","1941-0042"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Image Processing","raw_type":"journal-article"},{"id":"pmid:27019487","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/27019487","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on image processing : a publication of the IEEE Signal Processing Society","raw_type":null},{"id":"pmh:oai:arXiv.org:1307.0426","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1307.0426","pdf_url":"https://arxiv.org/pdf/1307.0426","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-04234016v1","is_oa":false,"landing_page_url":"https://hal.science/hal-04234016","pdf_url":null,"source":{"id":"https://openalex.org/S4406922466","display_name":"SPIRE - Sciences Po Institutional REpository","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Transactions on Image Processing, 2016, 25 (6), ⟨10.1109/TIP.2016.2544703⟩","raw_type":"Journal articles"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1307.0426","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1307.0426","pdf_url":"https://arxiv.org/pdf/1307.0426","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.6100000143051147}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W52210563","https://openalex.org/W86738361","https://openalex.org/W113453631","https://openalex.org/W273955616","https://openalex.org/W1518636435","https://openalex.org/W1565931986","https://openalex.org/W1593298603","https://openalex.org/W1967073774","https://openalex.org/W1968923340","https://openalex.org/W1976526581","https://openalex.org/W1978062184","https://openalex.org/W1986649315","https://openalex.org/W1998379572","https://openalex.org/W2010587020","https://openalex.org/W2063169554","https://openalex.org/W2065217067","https://openalex.org/W2066828176","https://openalex.org/W2074660466","https://openalex.org/W2082995931","https://openalex.org/W2083856558","https://openalex.org/W2084988058","https://openalex.org/W2085091083","https://openalex.org/W2086374833","https://openalex.org/W2089458919","https://openalex.org/W2097074406","https://openalex.org/W2100209835","https://openalex.org/W2101022607","https://openalex.org/W2102595307","https://openalex.org/W2103703342","https://openalex.org/W2110158442","https://openalex.org/W2118978333","https://openalex.org/W2119823327","https://openalex.org/W2121927366","https://openalex.org/W2122889464","https://openalex.org/W2123585888","https://openalex.org/W2124900523","https://openalex.org/W2124984765","https://openalex.org/W2128806031","https://openalex.org/W2132811257","https://openalex.org/W2134357733","https://openalex.org/W2134702142","https://openalex.org/W2143017915","https://openalex.org/W2144660879","https://openalex.org/W2145305441","https://openalex.org/W2148157540","https://openalex.org/W2148347694","https://openalex.org/W2156267451","https://openalex.org/W2162140684","https://openalex.org/W2163344010","https://openalex.org/W2163595993","https://openalex.org/W2164971516","https://openalex.org/W2165140157","https://openalex.org/W4388156124","https://openalex.org/W6610017368","https://openalex.org/W6671397441","https://openalex.org/W6680873320","https://openalex.org/W6681400661","https://openalex.org/W7037270093"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2051487156","https://openalex.org/W2073681303","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4295532600","https://openalex.org/W4396701345"],"abstract_inverted_index":{"Although":[0],"agreement":[1,92,107,112,132],"between":[2,143],"the":[3,15,27,34,63,81,89,94,144,163,172,177,188,191,218],"annotators":[4],"who":[5],"mark":[6],"feature":[7],"locations":[8],"within":[9],"images":[10],"has":[11,23],"been":[12],"studied":[13],"in":[14,46,65,113,208,229,238],"past":[16],"from":[17,57],"a":[18,114,140,154,166,205],"statistical":[19],"viewpoint,":[20],"little":[21],"work":[22],"attempted":[24],"to":[25,29,75,79,85,152,175,186,213,246],"quantify":[26,80],"extent":[28],"which":[30],"this":[31,53],"phenomenon":[32],"affects":[33,67],"evaluation":[35],"of":[36,96,156,165,190,217,232],"foreground-background":[37],"segmentation":[38,126],"algorithms.":[39],"Many":[40],"researchers":[41],"utilize":[42],"ground":[43],"truth":[44],"(GT)":[45],"experimentation":[47],"and":[48,84,93,133,179,183],"more":[49,225],"often":[50],"than":[51],"not":[52,244],"GT":[54,147],"is":[55,73,99,108,135,139,160,168,204,235,243],"derived":[56],"one":[58,216,255],"annotator's":[59],"opinion.":[60],"How":[61],"does":[62],"difference":[64],"opinion":[66],"an":[68,230,249],"algorithm's":[69],"evaluation?":[70],"A":[71],"methodology":[72],"applied":[74],"four":[76],"image-processing":[77],"problems":[78],"interannotator":[82],"variance":[83,207],"offer":[86],"insight":[87],"into":[88],"mechanisms":[90],"behind":[91],"use":[95],"GT.":[97,256],"It":[98,159,234],"found":[100,136,161],"that":[101,137,162,180,237],"when":[102,197,252],"detecting":[103],"linear":[104],"structures,":[105],"annotator":[106,131],"very":[109],"low.":[110],"The":[111],"structure's":[115],"position":[116],"can":[117],"be":[118],"partially":[119],"explained":[120],"through":[121],"basic":[122],"image":[123],"properties.":[124],"Automatic":[125],"algorithms":[127],"are":[128,150,199],"compared":[129],"with":[130],"it":[134,242],"there":[138,198,203],"clear":[141],"relation":[142],"two.":[145],"Several":[146],"estimation":[148],"methods":[149],"used":[151,174],"infer":[153,248],"number":[155],"algorithm":[157,250],"performances.":[158],"rank":[164],"detector":[167],"highly":[169],"dependent":[170],"upon":[171,254],"method":[173],"form":[176],"GT,":[178],"although":[181],"STAPLE":[182],"LSML":[184],"appear":[185],"represent":[187],"mean":[189],"performance":[192],"measured":[193],"using":[194],"individual":[195],"annotations,":[196,201],"few":[200],"or":[202],"large":[206],"them,":[209],"these":[210],"estimates":[211],"tend":[212],"degrade.":[214],"Furthermore,":[215],"most":[219],"commonly":[220],"adopted":[221],"combination":[222],"methods-consensus":[223],"voting-accentuates":[224],"obvious":[226],"features,":[227],"resulting":[228],"overestimation":[231],"performance.":[233],"concluded":[236],"some":[239],"data":[240],"sets,":[241],"possible":[245],"confidently":[247],"ranking":[251],"evaluating":[253]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":17},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":17},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":11},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":6},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":2},{"year":2013,"cited_by_count":1}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}