{"id":"https://openalex.org/W7133796410","doi":"https://doi.org/10.1145/3800939","title":"Human-centric Evaluation of Semantic Resources: A Systematic Mapping Study","display_name":"Human-centric Evaluation of Semantic Resources: A Systematic Mapping Study","publication_year":2026,"publication_date":"2026-03-05","ids":{"openalex":"https://openalex.org/W7133796410","doi":"https://doi.org/10.1145/3800939"},"language":"en","primary_location":{"id":"doi:10.1145/3800939","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3800939","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3800939","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5127319374","display_name":"Marta ; id_orcid 0000-0001-9301-8418 Sabou","orcid":null},"institutions":[{"id":"https://openalex.org/I102248843","display_name":"Vienna University of Economics and Business","ror":"https://ror.org/03yn8s215","country_code":"AT","type":"education","lineage":["https://openalex.org/I102248843"]}],"countries":["AT"],"is_corresponding":true,"raw_author_name":"Marta Sabou","raw_affiliation_strings":["Vienna University of Economics and Business"],"raw_orcid":"https://orcid.org/0000-0001-9301-8418","affiliations":[{"raw_affiliation_string":"Vienna University of Economics and Business","institution_ids":["https://openalex.org/I102248843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007529093","display_name":"Stefani Tsaneva","orcid":"https://orcid.org/0000-0002-0895-6379"},"institutions":[{"id":"https://openalex.org/I102248843","display_name":"Vienna University of Economics and Business","ror":"https://ror.org/03yn8s215","country_code":"AT","type":"education","lineage":["https://openalex.org/I102248843"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Stefani Tsaneva","raw_affiliation_strings":["Vienna University of Economics and Business"],"raw_orcid":"https://orcid.org/0000-0002-0895-6379","affiliations":[{"raw_affiliation_string":"Vienna University of Economics and Business","institution_ids":["https://openalex.org/I102248843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090994054","display_name":"Miriam Fern\u00e1ndez","orcid":"https://orcid.org/0000-0001-5939-4321"},"institutions":[{"id":"https://openalex.org/I204136569","display_name":"The Open University","ror":"https://ror.org/05mzfcs16","country_code":"GB","type":"education","lineage":["https://openalex.org/I204136569"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Miriam Fernandez","raw_affiliation_strings":["The Open University"],"raw_orcid":"https://orcid.org/0000-0001-5939-4321","affiliations":[{"raw_affiliation_string":"The Open University","institution_ids":["https://openalex.org/I204136569"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Mar\u00eda Poveda-Villal\u00f3n","orcid":"https://orcid.org/0000-0003-3587-0367"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Mar\u00eda Poveda-Villal\u00f3n","raw_affiliation_strings":["Universidad Polit\u00e9cnica de Madrid"],"raw_orcid":"https://orcid.org/0000-0003-3587-0367","affiliations":[{"raw_affiliation_string":"Universidad Polit\u00e9cnica de Madrid","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084762644","display_name":"Mari Carmen Su\u00e1rez-Figueroa","orcid":"https://orcid.org/0000-0003-3807-5019"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Mari Carmen Su\u00e1rez-Figueroa","raw_affiliation_strings":["Universidad Polit\u00e9cnica de Madrid"],"raw_orcid":"https://orcid.org/0000-0003-3807-5019","affiliations":[{"raw_affiliation_string":"Universidad Polit\u00e9cnica de Madrid","institution_ids":["https://openalex.org/I88060688"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5127319374"],"corresponding_institution_ids":["https://openalex.org/I102248843"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.3428263,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"58","issue":"10","first_page":"1","last_page":"38"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.605400025844574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.605400025844574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.19370000064373016,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.03440000116825104,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5041000247001648},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5016000270843506},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.48890000581741333},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.4083000123500824},{"id":"https://openalex.org/keywords/semantic-web","display_name":"Semantic Web","score":0.3652999997138977},{"id":"https://openalex.org/keywords/semantic-analysis","display_name":"Semantic analysis (machine learning)","score":0.3571000099182129},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.3495999872684479},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.34119999408721924}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8580999970436096},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5590999722480774},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5041000247001648},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5016000270843506},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.48890000581741333},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.4083000123500824},{"id":"https://openalex.org/C2129575","wikidata":"https://www.wikidata.org/wiki/Q54837","display_name":"Semantic Web","level":2,"score":0.3652999997138977},{"id":"https://openalex.org/C2777946921","wikidata":"https://www.wikidata.org/wiki/Q7449044","display_name":"Semantic analysis (machine learning)","level":2,"score":0.3571000099182129},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3495999872684479},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.34119999408721924},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3325999975204468},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.328000009059906},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.29750001430511475},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.28700000047683716},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2791999876499176},{"id":"https://openalex.org/C25810664","wikidata":"https://www.wikidata.org/wiki/Q44325","display_name":"Ontology","level":2,"score":0.2773999869823456},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.273499995470047},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2734000086784363},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.2694000005722046},{"id":"https://openalex.org/C90312973","wikidata":"https://www.wikidata.org/wiki/Q7449052","display_name":"Semantic data model","level":2,"score":0.2639999985694885},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.2606000006198883},{"id":"https://openalex.org/C3020001037","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assessment","level":3,"score":0.2563999891281128},{"id":"https://openalex.org/C102993220","wikidata":"https://www.wikidata.org/wiki/Q387196","display_name":"Description logic","level":2,"score":0.2549000084400177},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.25220000743865967}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3800939","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3800939","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},{"id":"pmh:oai:research.wu.ac.at:openaire/de1a0a06-a10e-40f1-bad9-06d974c53e6f","is_oa":true,"landing_page_url":"https://research.wu.ac.at/de/publications/de1a0a06-a10e-40f1-bad9-06d974c53e6f","pdf_url":null,"source":{"id":"https://openalex.org/S7407055123","display_name":"WU Research","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sabou, M, Tsaneva, S, Fernandez, M, Poveda-Vill\u00e1lon, M & Su\u00e1rez-Figueroa, M C 2026, 'Human-centric Evaluation of Semantic Resources: A Systematic Mapping Study', ACM Computing Surveys, vol. 58, no. 10, 270, pp. 1-38. https://doi.org/10.1145/3800939","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:oro.open.ac.uk:109349","is_oa":true,"landing_page_url":"https://oro.open.ac.uk/view/person/mff39.html>;","pdf_url":"https://oro.open.ac.uk/109349/9/109349.pdf","source":{"id":"https://openalex.org/S4306401187","display_name":"Open Research Online (The Open University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I204136569","host_organization_name":"The Open University","host_organization_lineage":["https://openalex.org/I204136569"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"},{"id":"pmh:doi:10.5281/zenodo.20199558","is_oa":true,"landing_page_url":"https://dataqualityws.github.io/","pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Presentation"}],"best_oa_location":{"id":"doi:10.1145/3800939","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3800939","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3435271498","display_name":null,"funder_award_id":"V 745-N","funder_id":"https://openalex.org/F4320321181","funder_display_name":"Austrian Science Fund"}],"funders":[{"id":"https://openalex.org/F4320321181","display_name":"Austrian Science Fund","ror":"https://ror.org/013tf3c58"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":113,"referenced_works":["https://openalex.org/W14427702","https://openalex.org/W67122473","https://openalex.org/W86426707","https://openalex.org/W141640963","https://openalex.org/W159127106","https://openalex.org/W166868646","https://openalex.org/W169904705","https://openalex.org/W323669384","https://openalex.org/W632432350","https://openalex.org/W1129276663","https://openalex.org/W1419282186","https://openalex.org/W1482164885","https://openalex.org/W1541456620","https://openalex.org/W1791587663","https://openalex.org/W1969536254","https://openalex.org/W1984761526","https://openalex.org/W1985049720","https://openalex.org/W1986165915","https://openalex.org/W1992269204","https://openalex.org/W1993471214","https://openalex.org/W1993516667","https://openalex.org/W1999798506","https://openalex.org/W2007868748","https://openalex.org/W2016245772","https://openalex.org/W2020740057","https://openalex.org/W2023844160","https://openalex.org/W2052978665","https://openalex.org/W2056141229","https://openalex.org/W2068154757","https://openalex.org/W2077442714","https://openalex.org/W2110961693","https://openalex.org/W2118328848","https://openalex.org/W2125382278","https://openalex.org/W2146736600","https://openalex.org/W2155249637","https://openalex.org/W2171450492","https://openalex.org/W2182894058","https://openalex.org/W2231771044","https://openalex.org/W2253965109","https://openalex.org/W2257040333","https://openalex.org/W2297849554","https://openalex.org/W2306611077","https://openalex.org/W2337312799","https://openalex.org/W2338050979","https://openalex.org/W2338709996","https://openalex.org/W2345053475","https://openalex.org/W2407412438","https://openalex.org/W2500230526","https://openalex.org/W2509883035","https://openalex.org/W2549536386","https://openalex.org/W2567173490","https://openalex.org/W2586809425","https://openalex.org/W2613512346","https://openalex.org/W2725428559","https://openalex.org/W2733135904","https://openalex.org/W2745405023","https://openalex.org/W2762043089","https://openalex.org/W2765565489","https://openalex.org/W2786943214","https://openalex.org/W2788646399","https://openalex.org/W2791800558","https://openalex.org/W2794771508","https://openalex.org/W2884063414","https://openalex.org/W2884315167","https://openalex.org/W2892449650","https://openalex.org/W2898634555","https://openalex.org/W2905118237","https://openalex.org/W2950862967","https://openalex.org/W2952100626","https://openalex.org/W2972459357","https://openalex.org/W2975412433","https://openalex.org/W2980901959","https://openalex.org/W2988709243","https://openalex.org/W2992437336","https://openalex.org/W3010336026","https://openalex.org/W3014641520","https://openalex.org/W3018814902","https://openalex.org/W3112515915","https://openalex.org/W3126308468","https://openalex.org/W3151685851","https://openalex.org/W3174440825","https://openalex.org/W3207508079","https://openalex.org/W3216389179","https://openalex.org/W4205317928","https://openalex.org/W4229038710","https://openalex.org/W4230177838","https://openalex.org/W4241904535","https://openalex.org/W4244643389","https://openalex.org/W4245861846","https://openalex.org/W4285818553","https://openalex.org/W4291213652","https://openalex.org/W4293722137","https://openalex.org/W4300950697","https://openalex.org/W4312065840","https://openalex.org/W4319057838","https://openalex.org/W4320926406","https://openalex.org/W4322576256","https://openalex.org/W4323662799","https://openalex.org/W4364374141","https://openalex.org/W4376612535","https://openalex.org/W4380204593","https://openalex.org/W4381381995","https://openalex.org/W4381665548","https://openalex.org/W4385571204","https://openalex.org/W4386532305","https://openalex.org/W4390875623","https://openalex.org/W4392596747","https://openalex.org/W4396936245","https://openalex.org/W4399694538","https://openalex.org/W4404724982","https://openalex.org/W4408276598","https://openalex.org/W4409309988","https://openalex.org/W6948681311"],"related_works":[],"abstract_inverted_index":{"Semantic":[0],"resources":[1,63],"such":[2,39,58],"as":[3,40,84,86,167,169],"ontologies":[4],"and":[5,51,93,111,146,151,154,165,174,185,194],"knowledge":[6],"graphs":[7],"enable":[8,179],"intelligent":[9],"applications":[10],"in":[11,78,135,141,182],"a":[12,69,72,81,106,116,125],"broad":[13],"range":[14],"of":[15,30,36,61,71,75,80,89,129,132],"application":[16],"areas.":[17],"As":[18],"ensuring":[19],"their":[20,31],"quality":[21],"is":[22,65,68],"important,":[23],"evaluation":[24,50,60],"activities":[25],"are":[26,45],"an":[27,87],"integral":[28],"part":[29],"life-cycle.":[32],"Yet,":[33],"several":[34],"aspects":[35],"these":[37],"resources,":[38],"domain":[41],"or":[42],"modelling":[43],"correctness,":[44],"not":[46],"amenable":[47],"to":[48,143,178,192],"automatic":[49],"require":[52],"assessment":[53],"through":[54],"human":[55],"participation.":[56],"Despite":[57],"human-centric":[59],"semantic":[62],"(HESR)":[64],"essential,":[66],"there":[67],"lack":[70],"systematic":[73],"understanding":[74],"this":[76,98,120,136,183],"area":[77,137,184],"terms":[79],"theoretical":[82,107,149,158],"framework":[83,108,150],"well":[85,168],"overview":[88],"concrete":[90],"HESR":[91,113,191],"approaches":[92],"trends":[94,153,164],".":[95],"We":[96],"address":[97],"gap":[99],"by,":[100],"firstly,":[101],"integrating":[102],"existing":[103],"literature":[104],"into":[105,162],"for":[109,189],"defining":[110],"characterising":[112],"thus":[114],"developing":[115],"new":[117,180],"perspective":[118],"on":[119],"area.":[121],"Second,":[122],"we":[123],"perform":[124],"Systematic":[126],"Mapping":[127],"Study":[128],"15":[130],"years":[131],"work":[133],"published":[134],"covering":[138],"144":[139],"papers":[140],"order":[142],"empirically":[144],"ground":[145],"extend":[147],"the":[148],"identify":[152],"best":[155],"practices.":[156],"These":[157],"foundations":[159],",":[160],"insights":[161],"current":[163],"practices":[166],"our":[170],"openly":[171],"available":[172],"data":[173],"analysis":[175],"scripts":[176],"aim":[177],"research":[181],"offer":[186],"practical":[187],"guidelines":[188],"performing":[190],"researchers":[193],"practitioners":[195],"alike.":[196]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2026-03-06T00:00:00"}