{"id":"https://openalex.org/W2512773859","doi":"https://doi.org/10.1007/978-3-319-44564-9_4","title":"Assessors Agreement: A Case Study Across Assessor Type, Payment Levels, Query Variations and Relevance Dimensions","display_name":"Assessors Agreement: A Case Study Across Assessor Type, Payment Levels, Query Variations and Relevance Dimensions","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2512773859","doi":"https://doi.org/10.1007/978-3-319-44564-9_4","mag":"2512773859"},"language":"en","primary_location":{"id":"doi:10.1007/978-3-319-44564-9_4","is_oa":false,"landing_page_url":"https://doi.org/10.1007/978-3-319-44564-9_4","pdf_url":null,"source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022129033","display_name":"Jo\u00e3o Palotti","orcid":"https://orcid.org/0000-0002-7099-9716"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Joao Palotti","raw_affiliation_strings":["Vienna University of Technology, Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Vienna University of Technology, Vienna, Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076031002","display_name":"Guido Zuccon","orcid":"https://orcid.org/0000-0003-0271-5563"},"institutions":[{"id":"https://openalex.org/I160993911","display_name":"Queensland University of Technology","ror":"https://ror.org/03pnv4752","country_code":"AU","type":"education","lineage":["https://openalex.org/I160993911"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Guido Zuccon","raw_affiliation_strings":["Queensland University of Technology, Brisbane, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Queensland University of Technology, Brisbane, Australia","institution_ids":["https://openalex.org/I160993911"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032071705","display_name":"Johannes Bernhardt","orcid":null},"institutions":[{"id":"https://openalex.org/I202276237","display_name":"Medical University of Graz","ror":"https://ror.org/02n0bts35","country_code":"AT","type":"education","lineage":["https://openalex.org/I202276237"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Johannes Bernhardt","raw_affiliation_strings":["Medical University of Graz, Graz, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Medical University of Graz, Graz, Austria","institution_ids":["https://openalex.org/I202276237"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020665735","display_name":"Allan Hanbury","orcid":"https://orcid.org/0000-0002-7149-5843"},"institutions":[{"id":"https://openalex.org/I145847075","display_name":"TU Wien","ror":"https://ror.org/04d836q62","country_code":"AT","type":"education","lineage":["https://openalex.org/I145847075"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Allan Hanbury","raw_affiliation_strings":["Vienna University of Technology, Vienna, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Vienna University of Technology, Vienna, Austria","institution_ids":["https://openalex.org/I145847075"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062228577","display_name":"Lorraine Goeuriot","orcid":"https://orcid.org/0000-0001-7491-1980"},"institutions":[{"id":"https://openalex.org/I899635006","display_name":"Universit\u00e9 Grenoble Alpes","ror":"https://ror.org/02rx3b187","country_code":"FR","type":"education","lineage":["https://openalex.org/I899635006"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Lorraine Goeuriot","raw_affiliation_strings":["Universit\u00e9 Grenoble Alpes, Saint-Martin-d\u2019H\u00e8res, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universit\u00e9 Grenoble Alpes, Saint-Martin-d\u2019H\u00e8res, France","institution_ids":["https://openalex.org/I899635006"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":5000,"currency":"EUR","value_usd":5392},"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"40","last_page":"53"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8503093123435974},{"id":"https://openalex.org/keywords/layperson","display_name":"Layperson","score":0.7870402932167053},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7149085998535156},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6336889863014221},{"id":"https://openalex.org/keywords/clef","display_name":"Clef","score":0.573885977268219},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5713698267936707},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5619938373565674},{"id":"https://openalex.org/keywords/cornerstone","display_name":"Cornerstone","score":0.5437946319580078},{"id":"https://openalex.org/keywords/payment","display_name":"Payment","score":0.512022078037262},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.43413493037223816},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.17742082476615906},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.08058139681816101}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8503093123435974},{"id":"https://openalex.org/C67597806","wikidata":"https://www.wikidata.org/wiki/Q15407657","display_name":"Layperson","level":2,"score":0.7870402932167053},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7149085998535156},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6336889863014221},{"id":"https://openalex.org/C107763842","wikidata":"https://www.wikidata.org/wiki/Q181040","display_name":"Clef","level":3,"score":0.573885977268219},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5713698267936707},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5619938373565674},{"id":"https://openalex.org/C2780616401","wikidata":"https://www.wikidata.org/wiki/Q1133673","display_name":"Cornerstone","level":2,"score":0.5437946319580078},{"id":"https://openalex.org/C145097563","wikidata":"https://www.wikidata.org/wiki/Q1148747","display_name":"Payment","level":2,"score":0.512022078037262},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.43413493037223816},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.17742082476615906},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.08058139681816101},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/978-3-319-44564-9_4","is_oa":false,"landing_page_url":"https://doi.org/10.1007/978-3-319-44564-9_4","pdf_url":null,"source":{"id":"https://openalex.org/S106296714","display_name":"Lecture notes in computer science","issn_l":"0302-9743","issn":["0302-9743","1611-3349"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"book series"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lecture Notes in Computer Science","raw_type":"book-chapter"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.46000000834465027,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W147182747","https://openalex.org/W1004439037","https://openalex.org/W1968927634","https://openalex.org/W2008285050","https://openalex.org/W2037140704","https://openalex.org/W2052989395","https://openalex.org/W2062236692","https://openalex.org/W2090031974","https://openalex.org/W2090035194","https://openalex.org/W2120308175","https://openalex.org/W2150937237","https://openalex.org/W2159937348","https://openalex.org/W2202662853","https://openalex.org/W2399777719","https://openalex.org/W2492868234","https://openalex.org/W3157166179"],"related_works":["https://openalex.org/W2021225351","https://openalex.org/W1839123017","https://openalex.org/W4232115401","https://openalex.org/W2905726621","https://openalex.org/W1592594506","https://openalex.org/W2970828463","https://openalex.org/W2320522151","https://openalex.org/W3138512925","https://openalex.org/W2069099855","https://openalex.org/W117735687"],"abstract_inverted_index":null,"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}