{"id":"https://openalex.org/W4403582516","doi":"https://doi.org/10.1145/3627673.3679174","title":"AnnoRank: A Comprehensive Web-Based Framework for Collecting Annotations and Assessing Rankings","display_name":"AnnoRank: A Comprehensive Web-Based Framework for Collecting Annotations and Assessing Rankings","publication_year":2024,"publication_date":"2024-10-20","ids":{"openalex":"https://openalex.org/W4403582516","doi":"https://doi.org/10.1145/3627673.3679174"},"language":"en","primary_location":{"id":"doi:10.1145/3627673.3679174","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627673.3679174","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3627673.3679174","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3627673.3679174","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024227030","display_name":"Clara Rus","orcid":"https://orcid.org/0000-0002-0944-5044"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Clara Rus","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048617828","display_name":"Gabrielle Poerwawinata","orcid":"https://orcid.org/0009-0002-4981-7292"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Gabrielle Poerwawinata","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059489981","display_name":"Andrew Yates","orcid":"https://orcid.org/0000-0002-5970-880X"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Andrew Yates","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031439294","display_name":"Maarten de Rijke","orcid":"https://orcid.org/0000-0002-1086-0202"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maarten de Rijke","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5024227030"],"corresponding_institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.28539734,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5400","last_page":"5404"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.983299970626831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.816156268119812},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.5506542921066284},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5142623782157898},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3511878550052643}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.816156268119812},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.5506542921066284},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5142623782157898},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3511878550052643}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3627673.3679174","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627673.3679174","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3627673.3679174","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},{"id":"pmh:oai:dare.uva.nl:openaire/b708051a-aac9-4adb-9d43-98198aaf5a97","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/annorank-a-comprehensive-webbased-framework-for-collecting-annotations-and-assessing-rankings(b708051a-aac9-4adb-9d43-98198aaf5a97).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Rus, C, Poerwawinata, G, Yates, A & de Rijke, M 2024, AnnoRank: A Comprehensive Web-Based Framework for Collecting Annotations and Assessing Rankings. in CIKM '24 : Proceedings of the 33rd ACM International Conference on Information and Knowledge Management : October, 21-25. 2024, Boise, ID, USA. Association for Computing Machinery, New York, NY, pp. 5400-5404, 33rd ACM International Conference on Information and Knowledge Management, CIKM 2024, Boise, United States, 21/10/24. https://doi.org/10.1145/3627673.3679174","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1145/3627673.3679174","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3627673.3679174","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3627673.3679174","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4403582516.pdf"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2108862644","https://openalex.org/W2120889650","https://openalex.org/W2133968343","https://openalex.org/W2143331230","https://openalex.org/W2151592910","https://openalex.org/W2152314154","https://openalex.org/W2159937348","https://openalex.org/W2296335794","https://openalex.org/W2704480242","https://openalex.org/W2798482016","https://openalex.org/W2964751853","https://openalex.org/W3094502663","https://openalex.org/W3102092462","https://openalex.org/W3103251620","https://openalex.org/W3152574220","https://openalex.org/W3152665210","https://openalex.org/W3156817235","https://openalex.org/W4294214983","https://openalex.org/W4320507976","https://openalex.org/W6603104078"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"We":[0,123],"present":[1],"AnnoRank,":[2],"a":[3,31,35,66,96,103,147],"web-based":[4],"user":[5,97],"interface":[6],"(UI)":[7],"framework":[8],"designed":[9,167],"to":[10,89,132,157,168,174,176],"facilitate":[11],"collecting":[12],"crowdsource":[13],"annotations":[14,29],"in":[15,83,102,162,181],"the":[16,23,42,48,61,84,91,99,107,113,126,134,137,143,158,163],"context":[17],"of":[18,25,44,50,65,93,110,128,136,150],"information":[19,182],"retrieval.":[20,183],"AnnoRank":[21,54,139,165,184],"enables":[22],"collection":[24],"explicit":[26],"and":[27,34,47,63,78,119,160,172],"implicit":[28],"for":[30,41,56,60],"specified":[32],"query":[33],"single":[36],"or":[37],"multiple":[38],"documents,":[39],"allowing":[40,59],"observation":[43],"user-selected":[45],"items":[46,101],"assignment":[49],"relevance":[51],"judgments.":[52],"Furthermore,":[53],"allows":[55],"ranking":[57,111,151],"comparisons,":[58],"visualization":[62],"evaluation":[64],"ranked":[67,104],"list":[68],"generated":[69],"by":[70],"different":[71],"fairness":[72,79],"interventions,":[73],"along":[74],"with":[75,142,192],"its":[76],"utility":[77],"metrics.":[80],"Fairness":[81],"interventions":[82],"annotation":[85,179],"pipeline":[86],"are":[87],"necessary":[88],"prevent":[90],"propagation":[92],"bias":[94],"when":[95],"selects":[98],"top-k":[100],"list.":[105],"With":[106],"widespread":[108],"use":[109],"systems,":[112],"application":[114],"supports":[115],"multimodality":[116],"through":[117],"text":[118],"image":[120],"document":[121],"formats.":[122],"also":[124],"support":[125],"assessment":[127],"agreement":[129],"between":[130],"annotators":[131],"ensure":[133],"quality":[135],"annotations.":[138],"is":[140,166,185],"integrated":[141],"Ranklib":[144],"library,":[145],"offering":[146],"vast":[148],"range":[149],"models":[152],"that":[153],"can":[154],"be":[155,169],"applied":[156],"data":[159],"displayed":[161],"UI.":[164],"flexible,":[170],"configurable,":[171],"easy":[173],"deploy":[175],"meet":[177],"diverse":[178],"needs":[180],"publicly":[186],"available":[187],"as":[188],"open-source":[189],"software,":[190],"together":[191],"detailed":[193],"documentation":[194],"at":[195],"https://github.com/ClaraRus/AnnoRank.":[196]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}