{"id":"https://openalex.org/W4408330908","doi":"https://doi.org/10.47989/ir30iconf47581","title":"Taking disagreements into consideration: human annotation variability in privacy policy analysis","display_name":"Taking disagreements into consideration: human annotation variability in privacy policy analysis","publication_year":2025,"publication_date":"2025-03-11","ids":{"openalex":"https://openalex.org/W4408330908","doi":"https://doi.org/10.47989/ir30iconf47581"},"language":"en","primary_location":{"id":"doi:10.47989/ir30iconf47581","is_oa":true,"landing_page_url":"https://doi.org/10.47989/ir30iconf47581","pdf_url":"https://publicera.kb.se/ir/article/download/47581/36925","source":{"id":"https://openalex.org/S4210195660","display_name":"Information Research an international electronic journal","issn_l":"1368-1613","issn":["1368-1613"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310317106","host_organization_name":"University of Bor\u00e5s","host_organization_lineage":["https://openalex.org/P4310317106"],"host_organization_lineage_names":["University of Bor\u00e5s"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Research an international electronic journal","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://publicera.kb.se/ir/article/download/47581/36925","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101444595","display_name":"Tian Wang","orcid":"https://orcid.org/0000-0001-8633-5001"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tian Wang","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042429203","display_name":"Yuanye Ma","orcid":"https://orcid.org/0000-0003-3007-2390"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuanye Ma","raw_affiliation_strings":["University of Illinois Discovery Partners Institute"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Discovery Partners Institute","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067502057","display_name":"Catherine Blake","orcid":"https://orcid.org/0000-0002-0600-629X"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Catherine Blake","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058187967","display_name":"Masooda Bashir","orcid":"https://orcid.org/0000-0001-8299-3111"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Masooda Bashir","raw_affiliation_strings":["University of Illinois Urbana-Champaign"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":null,"display_name":"Ryan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ryan Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.0646,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.88406165,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"30","issue":"iConf","first_page":"81","last_page":"92"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11045","display_name":"Privacy, Security, and Data Protection","score":0.9319000244140625,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11045","display_name":"Privacy, Security, and Data Protection","score":0.9319000244140625,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5030171275138855},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.497254878282547},{"id":"https://openalex.org/keywords/privacy-policy","display_name":"Privacy policy","score":0.43685382604599},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3798707127571106},{"id":"https://openalex.org/keywords/information-privacy","display_name":"Information privacy","score":0.3569856882095337},{"id":"https://openalex.org/keywords/internet-privacy","display_name":"Internet privacy","score":0.3206118941307068},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.07681071758270264}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5030171275138855},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.497254878282547},{"id":"https://openalex.org/C102938260","wikidata":"https://www.wikidata.org/wiki/Q1999831","display_name":"Privacy policy","level":3,"score":0.43685382604599},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3798707127571106},{"id":"https://openalex.org/C123201435","wikidata":"https://www.wikidata.org/wiki/Q456632","display_name":"Information privacy","level":2,"score":0.3569856882095337},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.3206118941307068},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.07681071758270264}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.47989/ir30iconf47581","is_oa":true,"landing_page_url":"https://doi.org/10.47989/ir30iconf47581","pdf_url":"https://publicera.kb.se/ir/article/download/47581/36925","source":{"id":"https://openalex.org/S4210195660","display_name":"Information Research an international electronic journal","issn_l":"1368-1613","issn":["1368-1613"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310317106","host_organization_name":"University of Bor\u00e5s","host_organization_lineage":["https://openalex.org/P4310317106"],"host_organization_lineage_names":["University of Bor\u00e5s"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Research an international electronic journal","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:e85b146489b84dbfa5c0f1b8f1ef5113","is_oa":true,"landing_page_url":"https://doaj.org/article/e85b146489b84dbfa5c0f1b8f1ef5113","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Information Research: An International Electronic Journal, Vol 30, Iss iConf (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.47989/ir30iconf47581","is_oa":true,"landing_page_url":"https://doi.org/10.47989/ir30iconf47581","pdf_url":"https://publicera.kb.se/ir/article/download/47581/36925","source":{"id":"https://openalex.org/S4210195660","display_name":"Information Research an international electronic journal","issn_l":"1368-1613","issn":["1368-1613"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310317106","host_organization_name":"University of Bor\u00e5s","host_organization_lineage":["https://openalex.org/P4310317106"],"host_organization_lineage_names":["University of Bor\u00e5s"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Research an international electronic journal","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4408330908.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2116878667","https://openalex.org/W2111170006","https://openalex.org/W2528826491","https://openalex.org/W3042284153","https://openalex.org/W4241527182","https://openalex.org/W3038106605","https://openalex.org/W2260440995","https://openalex.org/W3195097297","https://openalex.org/W2035493462","https://openalex.org/W2166647055"],"abstract_inverted_index":{"Introduction.":[0],"Privacy":[1],"policies":[2],"inform":[3],"users":[4],"about":[5],"data":[6],"practices":[7],"but":[8],"are":[9],"often":[10],"complex":[11],"and":[12,44,68,92,121,179],"difficult":[13],"to":[14,74,161,174],"interpret.":[15],"Human":[16],"annotation":[17,27,42,54,135,139,154,172],"plays":[18],"a":[19,157],"key":[20],"role":[21],"in":[22],"understanding":[23,134],"privacy":[24,162],"policies,":[25],"yet":[26],"disagreements":[28,55,125,140,173],"highlight":[29],"the":[30,61,113,131,169],"complexity":[31],"of":[32,171],"these":[33],"texts.":[34],"Traditional":[35],"machine":[36,57,176],"learning":[37,58,177],"models":[38],"prioritize":[39],"consensus,":[40],"overlooking":[41],"variability":[43],"its":[45],"impact":[46,77,127],"on":[47,78],"accuracy.":[48,144],"Method.":[49],"This":[50,145],"study":[51,83,146],"examines":[52],"how":[53],"affect":[56],"performance":[59,106],"using":[60],"OPP-115":[62],"corpus.":[63],"It":[64],"compares":[65],"majority":[66],"vote":[67],"union":[69],"methods":[70],"with":[71],"alternative":[72],"strategies":[73,150],"assess":[75],"their":[76],"policy":[79,163],"classification.":[80],"Analysis.":[81],"The":[82],"evaluates":[84],"whether":[85],"increasing":[86],"annotator":[87],"consensus":[88],"improves":[89],"model":[90,105,143],"effectiveness":[91],"if":[93],"disagreement-aware":[94],"approaches":[95],"yield":[96],"more":[97,158],"reliable":[98],"results.":[99],"Results.":[100],"Higher":[101],"agreement":[102,111],"levels":[103],"improve":[104,175],"across":[107],"most":[108],"categories.":[109],"Complete":[110],"yields":[112],"best":[114],"F1-scores,":[115],"especially":[116],"for":[117,133,153],"First":[118],"Party":[119],"Collection/Use":[120],"Third-Party":[122],"Sharing/Collection.":[123],"Annotation":[124],"significantly":[126],"classification":[128],"outcomes,":[129],"underscoring":[130],"need":[132],"disagreements.":[136],"Conclusions.":[137],"Ignoring":[138],"can":[141],"misrepresent":[142],"proposes":[147],"new":[148],"evaluation":[149],"that":[151],"account":[152],"variability,":[155],"offering":[156],"realistic":[159],"approach":[160],"analysis.":[164],"Future":[165],"work":[166],"should":[167],"explore":[168],"causes":[170],"transparency":[178],"reliability.":[180]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}