{"id":"https://openalex.org/W1852596290","doi":"https://doi.org/10.1145/1743384.1743398","title":"Performance measures for multilabel evaluation","display_name":"Performance measures for multilabel evaluation","publication_year":2010,"publication_date":"2010-03-29","ids":{"openalex":"https://openalex.org/W1852596290","doi":"https://doi.org/10.1145/1743384.1743398","mag":"1852596290"},"language":"en","primary_location":{"id":"doi:10.1145/1743384.1743398","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1743384.1743398","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the international conference on Multimedia information retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005712172","display_name":"Stefanie Nowak","orcid":null},"institutions":[{"id":"https://openalex.org/I4210138578","display_name":"Fraunhofer Institute for Digital Media Technology","ror":"https://ror.org/04gp0yb49","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210138578","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Stefanie Nowak","raw_affiliation_strings":["Fraunhofer IDMT, Ilmenau, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer IDMT, Ilmenau, Germany","institution_ids":["https://openalex.org/I4210138578"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067411318","display_name":"Hanna Lukashevich","orcid":"https://orcid.org/0000-0002-9041-9475"},"institutions":[{"id":"https://openalex.org/I4210138578","display_name":"Fraunhofer Institute for Digital Media Technology","ror":"https://ror.org/04gp0yb49","country_code":"DE","type":"facility","lineage":["https://openalex.org/I4210138578","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Hanna Lukashevich","raw_affiliation_strings":["Fraunhofer IDMT, Ilmenau, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer IDMT, Ilmenau, Germany","institution_ids":["https://openalex.org/I4210138578"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009503318","display_name":"Peter Dunker","orcid":null},"institutions":[{"id":"https://openalex.org/I169912796","display_name":"Grace (United States)","ror":"https://ror.org/03k4vk194","country_code":"US","type":"company","lineage":["https://openalex.org/I169912796"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter Dunker","raw_affiliation_strings":["Gracenote, Inc, Emeryville, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Gracenote, Inc, Emeryville, CA, USA","institution_ids":["https://openalex.org/I169912796"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112279721","display_name":"Stefan R\u00fcger","orcid":null},"institutions":[{"id":"https://openalex.org/I204136569","display_name":"The Open University","ror":"https://ror.org/05mzfcs16","country_code":"GB","type":"education","lineage":["https://openalex.org/I204136569"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Stefan R\u00fcger","raw_affiliation_strings":["Open University, Milton Keynes, England UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Open University, Milton Keynes, England UK","institution_ids":["https://openalex.org/I204136569"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2719,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.40507934,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"35","last_page":"44"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9819999933242798,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8055627346038818},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7398317456245422},{"id":"https://openalex.org/keywords/ontology","display_name":"Ontology","score":0.6736629605293274},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.632444441318512},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5546332597732544},{"id":"https://openalex.org/keywords/search-engine-indexing","display_name":"Search engine indexing","score":0.5072271823883057},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5031453967094421},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.48400089144706726},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.477665513753891},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.40814658999443054},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31986886262893677},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1062883734703064}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8055627346038818},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7398317456245422},{"id":"https://openalex.org/C25810664","wikidata":"https://www.wikidata.org/wiki/Q44325","display_name":"Ontology","level":2,"score":0.6736629605293274},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.632444441318512},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5546332597732544},{"id":"https://openalex.org/C75165309","wikidata":"https://www.wikidata.org/wiki/Q2258979","display_name":"Search engine indexing","level":2,"score":0.5072271823883057},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5031453967094421},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.48400089144706726},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.477665513753891},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.40814658999443054},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31986886262893677},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1062883734703064},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/1743384.1743398","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1743384.1743398","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the international conference on Multimedia information retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:oro.open.ac.uk:25871","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401187","display_name":"Open Research Online (The Open University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I204136569","host_organization_name":"The Open University","host_organization_lineage":["https://openalex.org/I204136569"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"Conference or Workshop Item"},{"id":"mag:1852596290","is_oa":false,"landing_page_url":"http://oro.open.ac.uk/25871/","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.49000000953674316,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1659833910","https://openalex.org/W1891372937","https://openalex.org/W1953606363","https://openalex.org/W2027257218","https://openalex.org/W2099007647","https://openalex.org/W2100567627","https://openalex.org/W2127563440","https://openalex.org/W2135293953","https://openalex.org/W2142549486","https://openalex.org/W2155803963","https://openalex.org/W2158394179","https://openalex.org/W2163533183","https://openalex.org/W2406621542","https://openalex.org/W2497384426","https://openalex.org/W3015740509","https://openalex.org/W6636265348","https://openalex.org/W6674596984","https://openalex.org/W6679387137","https://openalex.org/W6685998662","https://openalex.org/W6728200263","https://openalex.org/W6983592330","https://openalex.org/W7036679014"],"related_works":["https://openalex.org/W2003497265","https://openalex.org/W69529220","https://openalex.org/W1981888042","https://openalex.org/W2951271148","https://openalex.org/W3105864483","https://openalex.org/W1970079616","https://openalex.org/W2937170550","https://openalex.org/W116336358","https://openalex.org/W2263945089","https://openalex.org/W2041353081","https://openalex.org/W2969782064","https://openalex.org/W1561561947","https://openalex.org/W178163987","https://openalex.org/W2352397247","https://openalex.org/W2399635203","https://openalex.org/W2897642955","https://openalex.org/W2015217008","https://openalex.org/W2997030765","https://openalex.org/W2304524576","https://openalex.org/W3157271831"],"abstract_inverted_index":{"With":[0],"the":[1,9,14,42,55,70,80,108,111,127,140,148,152,169],"steadily":[2],"increasing":[3],"amount":[4],"of":[5,31,41,57,171],"multimedia":[6],"documents":[7],"on":[8,50,92],"web":[10],"and":[11,62,67,110,118,168],"at":[12],"home,":[13],"need":[15],"for":[16,115,126,163],"reliable":[17],"semantic":[18],"indexing":[19],"methods":[20],"that":[21,77,102,142],"assign":[22,136],"multiple":[23],"keywords":[24],"to":[25,120,139],"a":[26,47,116,121],"document":[27],"grows.":[28],"The":[29,124,133,174],"performance":[30],"existing":[32],"approaches":[33],"is":[34],"often":[35],"measured":[36],"with":[37,88],"standard":[38],"evaluation":[39,60,179],"measures":[40,61,150],"information":[43],"retrieval":[44],"community.":[45],"In":[46],"case":[48,182],"study":[49],"image":[51],"annotation,":[52],"we":[53],"show":[54],"behaviour":[56],"13":[58],"different":[59],"point":[63],"out":[64],"their":[65],"strengths":[66],"weaknesses.":[68],"For":[69,155],"analysis,":[71],"data":[72],"from":[73,107],"19":[74],"research":[75],"groups":[76],"participated":[78],"in":[79,147,181],"ImageCLEF":[81],"Photo":[82],"Annotation":[83],"Task":[84],"are":[85,130],"utilized":[86],"together":[87],"several":[89],"configurations":[90],"based":[91],"random":[93,166],"numbers.":[94],"A":[95],"recently":[96],"proposed":[97],"ontology-based":[98,134],"measure":[99,129,176],"was":[100],"investigated":[101],"incorporates":[103],"structure":[104],"information,":[105],"relationships":[106],"ontology":[109],"agreement":[112],"between":[113],"annotators":[114],"concept":[117],"compared":[119],"hierarchical":[122,128],"variant.":[123],"results":[125,135,159],"not":[131],"competitive.":[132],"good":[137,145,178],"scores":[138],"systems":[141],"got":[143],"also":[144],"ranks":[146],"other":[149],"like":[151],"example-based":[153],"F-measure.":[154],"concept-based":[156],"evaluation,":[157],"stable":[158],"could":[160],"be":[161],"obtained":[162],"MAP":[164],"concerning":[165],"numbers":[167],"number":[170],"annotated":[172],"labels.":[173],"AUC":[175],"shows":[177],"characteristics":[180],"all":[183],"annotations":[184],"contain":[185],"confidence":[186],"values.":[187]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}