{"id":"https://openalex.org/W4404288914","doi":"https://doi.org/10.1007/s10506-024-09423-9","title":"Deciphering disagreement in the annotation of EU legislation","display_name":"Deciphering disagreement in the annotation of EU legislation","publication_year":2024,"publication_date":"2024-11-12","ids":{"openalex":"https://openalex.org/W4404288914","doi":"https://doi.org/10.1007/s10506-024-09423-9"},"language":"en","primary_location":{"id":"doi:10.1007/s10506-024-09423-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10506-024-09423-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10506-024-09423-9.pdf","source":{"id":"https://openalex.org/S96609033","display_name":"Artificial Intelligence and Law","issn_l":"0924-8463","issn":["0924-8463","1572-8382"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Artificial Intelligence and Law","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10506-024-09423-9.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054598855","display_name":"Gijs van Dijck","orcid":"https://orcid.org/0000-0003-4102-4415"},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Gijs van Dijck","raw_affiliation_strings":["Brightlands Institute for Smart Society, Maastricht University, Maastricht, The Netherlands","Maastricht Law and Tech Lab, Maastricht University, Maastricht, The Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-4102-4415","affiliations":[{"raw_affiliation_string":"Brightlands Institute for Smart Society, Maastricht University, Maastricht, The Netherlands","institution_ids":["https://openalex.org/I34352273"]},{"raw_affiliation_string":"Maastricht Law and Tech Lab, Maastricht University, Maastricht, The Netherlands","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109905389","display_name":"Carlos Aguilera","orcid":null},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Carlos Aguilera","raw_affiliation_strings":["Brightlands Institute for Smart Society, Maastricht University, Maastricht, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brightlands Institute for Smart Society, Maastricht University, Maastricht, The Netherlands","institution_ids":["https://openalex.org/I34352273"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102982829","display_name":"Shashank M. Chakravarthy","orcid":"https://orcid.org/0000-0003-3336-8402"},"institutions":[{"id":"https://openalex.org/I34352273","display_name":"Maastricht University","ror":"https://ror.org/02jz4aj89","country_code":"NL","type":"education","lineage":["https://openalex.org/I34352273"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Shashank M. Chakravarthy","raw_affiliation_strings":["Brightlands Institute for Smart Society, Maastricht University, Maastricht, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brightlands Institute for Smart Society, Maastricht University, Maastricht, The Netherlands","institution_ids":["https://openalex.org/I34352273"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5054598855"],"corresponding_institution_ids":["https://openalex.org/I34352273"],"apc_list":{"value":2390,"currency":"EUR","value_usd":2990},"apc_paid":{"value":2390,"currency":"EUR","value_usd":2990},"fwci":2.0183,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.90000059,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"34","issue":"1","first_page":"191","last_page":"226"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13710","display_name":"European and International Law Studies","score":0.9595000147819519,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13710","display_name":"European and International Law Studies","score":0.9595000147819519,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12921","display_name":"European Criminal Justice and Data Protection","score":0.9444000124931335,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/legislation","display_name":"Legislation","score":0.7056350708007812},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.6539051532745361},{"id":"https://openalex.org/keywords/legal-aspects-of-computing","display_name":"Legal aspects of computing","score":0.5237317681312561},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.5054969787597656},{"id":"https://openalex.org/keywords/philosophy-of-law","display_name":"Philosophy of law","score":0.46980103850364685},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3951200544834137},{"id":"https://openalex.org/keywords/law-and-economics","display_name":"Law and economics","score":0.36723750829696655},{"id":"https://openalex.org/keywords/internet-privacy","display_name":"Internet privacy","score":0.34368735551834106},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.33698147535324097},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.309853732585907},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.2947656214237213},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2887500524520874},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2743694484233856},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.1825893223285675},{"id":"https://openalex.org/keywords/comparative-law","display_name":"Comparative law","score":0.13092386722564697}],"concepts":[{"id":"https://openalex.org/C2777351106","wikidata":"https://www.wikidata.org/wiki/Q49371","display_name":"Legislation","level":2,"score":0.7056350708007812},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6539051532745361},{"id":"https://openalex.org/C67497173","wikidata":"https://www.wikidata.org/wiki/Q977345","display_name":"Legal aspects of computing","level":3,"score":0.5237317681312561},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.5054969787597656},{"id":"https://openalex.org/C162040827","wikidata":"https://www.wikidata.org/wiki/Q126842","display_name":"Philosophy of law","level":3,"score":0.46980103850364685},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3951200544834137},{"id":"https://openalex.org/C190253527","wikidata":"https://www.wikidata.org/wiki/Q295354","display_name":"Law and economics","level":1,"score":0.36723750829696655},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.34368735551834106},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.33698147535324097},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.309853732585907},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.2947656214237213},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2887500524520874},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2743694484233856},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.1825893223285675},{"id":"https://openalex.org/C149209484","wikidata":"https://www.wikidata.org/wiki/Q741338","display_name":"Comparative law","level":2,"score":0.13092386722564697}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s10506-024-09423-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10506-024-09423-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10506-024-09423-9.pdf","source":{"id":"https://openalex.org/S96609033","display_name":"Artificial Intelligence and Law","issn_l":"0924-8463","issn":["0924-8463","1572-8382"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Artificial Intelligence and Law","raw_type":"journal-article"},{"id":"pmh:oai:cris.maastrichtuniversity.nl:openaire_cris_publications/1ce8bd43-59b4-46c6-b7b3-71f02e3fd51c","is_oa":true,"landing_page_url":"https://cris.maastrichtuniversity.nl/en/publications/1ce8bd43-59b4-46c6-b7b3-71f02e3fd51c","pdf_url":null,"source":{"id":"https://openalex.org/S4306402616","display_name":"Research Publications (Maastricht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I34352273","host_organization_name":"Maastricht University","host_organization_lineage":["https://openalex.org/I34352273"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"van Dijck, G, Aguilera Medina, C & Chakravarthy, S 2026, 'Deciphering disagreement in the annotation of EU legislation', Artificial Intelligence and Law, vol. 34, no. 1, pp. 191-226. https://doi.org/10.1007/s10506-024-09423-9","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1007/s10506-024-09423-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10506-024-09423-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10506-024-09423-9.pdf","source":{"id":"https://openalex.org/S96609033","display_name":"Artificial Intelligence and Law","issn_l":"0924-8463","issn":["0924-8463","1572-8382"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Artificial Intelligence and Law","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.6200000047683716}],"awards":[{"id":"https://openalex.org/G222375268","display_name":"Creating the 21st century spatial ecosystem","funder_award_id":"101006828","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G3176715082","display_name":null,"funder_award_id":"101006828-Flying Forward 2020","funder_id":"https://openalex.org/F4320332999","funder_display_name":"Horizon 2020 Framework Programme"},{"id":"https://openalex.org/G4937468798","display_name":null,"funder_award_id":"H2020","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320332999","display_name":"Horizon 2020 Framework Programme","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404288914.pdf","grobid_xml":"https://content.openalex.org/works/W4404288914.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W16123658","https://openalex.org/W1606205296","https://openalex.org/W1752934815","https://openalex.org/W1825943944","https://openalex.org/W1975879668","https://openalex.org/W1979714143","https://openalex.org/W2015282105","https://openalex.org/W2035799785","https://openalex.org/W2037789405","https://openalex.org/W2038293487","https://openalex.org/W2053154970","https://openalex.org/W2121882970","https://openalex.org/W2164777277","https://openalex.org/W2750019983","https://openalex.org/W2757079888","https://openalex.org/W2883086394","https://openalex.org/W2896718675","https://openalex.org/W2897549266","https://openalex.org/W2963829982","https://openalex.org/W3038062918","https://openalex.org/W3092093993","https://openalex.org/W3136275640","https://openalex.org/W3194623849","https://openalex.org/W4210764005","https://openalex.org/W4240853199","https://openalex.org/W4287888314","https://openalex.org/W4293694313","https://openalex.org/W4311978845","https://openalex.org/W4322486700","https://openalex.org/W4362723303","https://openalex.org/W4382280470","https://openalex.org/W4389518785"],"related_works":["https://openalex.org/W3122213275","https://openalex.org/W1981013049","https://openalex.org/W2080633635","https://openalex.org/W2065068111","https://openalex.org/W2753556841","https://openalex.org/W1606005905","https://openalex.org/W2187027340","https://openalex.org/W2938644490","https://openalex.org/W2337751453","https://openalex.org/W3024612645"],"abstract_inverted_index":{"Abstract":[0],"The":[1,85,103,143,171,261],"topic":[2],"of":[3,15,110,154,160,165,222,224,230,257,266,281],"annotating":[4,258],"legal":[5],"data":[6],"has":[7],"received":[8],"surprisingly":[9],"little":[10],"attention.":[11],"A":[12],"key":[13],"challenge":[14],"the":[16,79,100,108,152,158,161,163,168,225,255,264,278],"annotation":[17,109,283],"process":[18],"is":[19,105],"reaching":[20],"a":[21,68,92,220,228],"sufficient":[22],"agreement":[23,54,59,73,87,147,172,242],"between":[24,61,207],"annotators":[25,62,76,248],"and":[26,41,56,66,71,122,137,167,269,276],"filtering":[27],"mistakes":[28],"from":[29,219],"genuine":[30],"disagreement.":[31,200],"This":[32,192,212],"study":[33],"presents":[34],"an":[35,97,282],"approach":[36],"that":[37,146,194,247],"provides":[38],"insights":[39],"into":[40],"resolves":[42],"potential":[43],"disagreement":[44],"amongst":[45],"annotators.":[46,142,211],"It":[47],"(1)":[48],"introduces":[49],"different":[50,274],"strategies":[51],"to":[52,91,217,239],"calculate":[53],"levels":[55,60,74,88,148],"compares":[57],"(2)":[58],"(inter-annotator":[63],"agreement)":[64],"before":[65],"after":[67,119,188],"revision":[69,93,190],"round":[70,94],"(3)":[72],"for":[75,123],"who":[77],"annotate":[78],"same":[80],"texts":[81],"twice":[82],"(intra-annotator":[83],"agreement).":[84],"inter-annotator":[86,241],"are":[89],"compared":[90],"in":[95,183,232,273],"which":[96,253],"arbiter":[98],"corrected":[99],"annotator\u2019s":[101],"labels.":[102],"analysis":[104],"based":[106,150],"on":[107,151],"EU":[111],"legislative":[112,259],"provisions":[113],"at":[114,277],"two":[115],"stages":[116,280],"(initial":[117],"annotations,":[118],"annotator":[120,169,195],"revisions)":[121],"various":[124,279],"tasks":[125],"(Definitions,":[126],"References,":[127],"Quantities,":[128],"IF-THEN":[129],"statements,":[130],"Exceptions,":[131],"Scope,":[132],"Hierarchy,":[133],"Deontic":[134],"Clauses,":[135],"Active":[136],"Passive":[138],"Role)":[139],"by":[140,237],"multiple":[141,250],"results":[144,262],"reveal":[145],"vary":[149],"stage":[153],"measurement":[155],"(before/after":[156],"revisions),":[157],"nature":[159],"task,":[162],"method":[164],"assessment,":[166],"combination.":[170],"scores":[173],"-":[174],"along":[175],"with":[176,180],"some":[177],"initial":[178],"measurements\u2014align":[179],"those":[181],"reported":[182],"previous":[184],"research":[185],"but":[186,208],"increase":[187],"each":[189],"round.":[191],"suggests":[193],"revisions":[196],"can":[197],"substantially":[198],"reduce":[199],"Additionally,":[201],"disagreements":[202],"were":[203],"found":[204],"not":[205,215],"only":[206],"also":[209],"among":[210],"inconsistency":[213],"does":[214],"appear":[216],"stem":[218],"lack":[221,229],"understanding":[223],"guidelines":[226],"or":[227],"seriousness":[231],"task":[233],"execution,":[234],"as":[235],"evidenced":[236],"moderate":[238],"substantial":[240],"scores.":[243],"These":[244],"findings":[245],"suggest":[246],"identified":[249],"valid":[251],"interpretations,":[252],"highlights":[254],"complexity":[256],"provisions.":[260],"underscore":[263],"significance":[265],"embracing,":[267],"addressing,":[268],"reporting":[270],"about":[271],"(dis)agreement":[272],"ways":[275],"task.":[284]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-05-21T09:19:25.381259","created_date":"2025-10-10T00:00:00"}