{"id":"https://openalex.org/W4381746764","doi":"https://doi.org/10.1109/percomworkshops56833.2023.10150351","title":"When the Ground Truth is not True: Modelling Human Biases in Temporal Annotations","display_name":"When the Ground Truth is not True: Modelling Human Biases in Temporal Annotations","publication_year":2023,"publication_date":"2023-03-13","ids":{"openalex":"https://openalex.org/W4381746764","doi":"https://doi.org/10.1109/percomworkshops56833.2023.10150351"},"language":"en","primary_location":{"id":"doi:10.1109/percomworkshops56833.2023.10150351","is_oa":false,"landing_page_url":"https://doi.org/10.1109/percomworkshops56833.2023.10150351","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Pervasive Computing and Communications Workshops and other Affiliated Events (PerCom Workshops)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043261884","display_name":"Taku Yamagata","orcid":"https://orcid.org/0000-0003-2122-0929"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Taku Yamagata","raw_affiliation_strings":["University of Bristol,United Kingdom","University of Bristol, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Bristol,United Kingdom","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048041586","display_name":"Emma L. Tonkin","orcid":"https://orcid.org/0000-0001-7405-4982"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Emma L. Tonkin","raw_affiliation_strings":["University of Bristol,United Kingdom","University of Bristol, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Bristol,United Kingdom","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001127008","display_name":"Benjamin Arana Sanchez","orcid":null},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Benjamin Arana Sanchez","raw_affiliation_strings":["University of Bristol,United Kingdom","University of Bristol, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Bristol,United Kingdom","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062152110","display_name":"Ian Craddock","orcid":"https://orcid.org/0000-0001-6552-8541"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ian Craddock","raw_affiliation_strings":["University of Bristol,United Kingdom","University of Bristol, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Bristol,United Kingdom","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022292656","display_name":"Miquel Perell\u00f3-Nieto","orcid":"https://orcid.org/0000-0001-8925-424X"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Miquel Perello Nieto","raw_affiliation_strings":["University of Bristol,United Kingdom","University of Bristol, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Bristol,United Kingdom","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028764929","display_name":"Ra\u00fal Santos\u2010Rodr\u00edguez","orcid":"https://orcid.org/0000-0001-9576-3905"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Raul Santos-Rodriguez","raw_affiliation_strings":["University of Bristol,United Kingdom","University of Bristol, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Bristol,United Kingdom","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100844899","display_name":"Weisong Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Weisong Yang","raw_affiliation_strings":["University of Bristol,United Kingdom","University of Bristol, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Bristol,United Kingdom","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057936560","display_name":"Peter Flach","orcid":"https://orcid.org/0000-0001-6857-5810"},"institutions":[{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Peter Flach","raw_affiliation_strings":["University of Bristol,United Kingdom","University of Bristol, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Bristol,United Kingdom","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"University of Bristol, United Kingdom","institution_ids":["https://openalex.org/I36234482"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5043261884"],"corresponding_institution_ids":["https://openalex.org/I36234482"],"apc_list":null,"apc_paid":null,"fwci":0.201,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.41499969,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"527","last_page":"533"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10444","display_name":"Context-Aware Activity Recognition Systems","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9878000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.8371737003326416},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.8166940808296204},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7684582471847534},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5717334151268005},{"id":"https://openalex.org/keywords/rendering","display_name":"Rendering (computer graphics)","score":0.5515843629837036},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.5474085211753845},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5026257038116455},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4993247985839844},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4396139085292816},{"id":"https://openalex.org/keywords/rounding","display_name":"Rounding","score":0.4327653646469116},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3852687478065491}],"concepts":[{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.8371737003326416},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.8166940808296204},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7684582471847534},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5717334151268005},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.5515843629837036},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.5474085211753845},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5026257038116455},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4993247985839844},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4396139085292816},{"id":"https://openalex.org/C136625980","wikidata":"https://www.wikidata.org/wiki/Q663208","display_name":"Rounding","level":2,"score":0.4327653646469116},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3852687478065491},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/percomworkshops56833.2023.10150351","is_oa":false,"landing_page_url":"https://doi.org/10.1109/percomworkshops56833.2023.10150351","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE International Conference on Pervasive Computing and Communications Workshops and other Affiliated Events (PerCom Workshops)","raw_type":"proceedings-article"},{"id":"pmh:oai:research-information.bris.ac.uk:openaire_cris_publications/8065e607-6f9f-42f2-bcbd-1b3a3681e6de","is_oa":false,"landing_page_url":"https://research-information.bris.ac.uk/en/publications/8065e607-6f9f-42f2-bcbd-1b3a3681e6de","pdf_url":null,"source":{"id":"https://openalex.org/S7407055359","display_name":"Explore Bristol Research","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Yamagata, T, Tonkin, E L, Arana Sanchez, B, Craddock, I J, Perello Nieto, M, Santos-Rodriguez, R, Yang, W & Flach, P A 2023, 'When the Ground Truth is not True: Modelling Human Biases in Temporal Annotations'. https://doi.org/10.1109/percomworkshops56833.2023.10150351","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.550000011920929}],"awards":[{"id":"https://openalex.org/G3315974449","display_name":"SPHERE - A Sensor Platform for HEalthcare in a Residential Environment (IRC Next Steps)","funder_award_id":"EP/R005273/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G7396137307","display_name":null,"funder_award_id":"EP/R005273/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G7629027287","display_name":null,"funder_award_id":"EP/V024817/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G7935266708","display_name":null,"funder_award_id":"EP/V024817/1","funder_id":"https://openalex.org/F4320314731","funder_display_name":"UK Research and Innovation"}],"funders":[{"id":"https://openalex.org/F4320314731","display_name":"UK Research and Innovation","ror":"https://ror.org/001aqnf71"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W103029643","https://openalex.org/W1897824230","https://openalex.org/W1968655745","https://openalex.org/W1975721533","https://openalex.org/W1983802226","https://openalex.org/W2003441971","https://openalex.org/W2052638004","https://openalex.org/W2066903463","https://openalex.org/W2080651358","https://openalex.org/W2093091956","https://openalex.org/W2187515986","https://openalex.org/W2264170626","https://openalex.org/W2357329924","https://openalex.org/W2763172196","https://openalex.org/W2884114130","https://openalex.org/W2971670291","https://openalex.org/W3017933464","https://openalex.org/W3165201394","https://openalex.org/W4212774754","https://openalex.org/W4229049024","https://openalex.org/W6604222146","https://openalex.org/W6681102160","https://openalex.org/W6791720863"],"related_works":["https://openalex.org/W4220780102","https://openalex.org/W2410881844","https://openalex.org/W3196334750","https://openalex.org/W1502401885","https://openalex.org/W2004257129","https://openalex.org/W2116281088","https://openalex.org/W2931688134","https://openalex.org/W2357551824","https://openalex.org/W2016668641","https://openalex.org/W2743738614"],"abstract_inverted_index":{"In":[0,52],"supervised":[1],"learning,":[2],"low":[3],"quality":[4,28],"annotations":[5,97],"lead":[6],"to":[7,83,91],"poorly":[8],"performing":[9],"classification":[10],"and":[11,57,78,98],"detection":[12],"models,":[13],"while":[14],"also":[15],"rendering":[16],"evaluation":[17],"unreliable.":[18],"This":[19],"is":[20,29],"particularly":[21],"apparent":[22],"on":[23,95,130],"temporal":[24,96],"data,":[25],"where":[26],"annotation":[27],"affected":[30],"by":[31,69],"multiple":[32],"factors.":[33],"For":[34],"example,":[35],"in":[36,108],"the":[37,48,55,76,101,120,128],"post-hoc":[38],"self-reporting":[39],"of":[40,47,59,80,103,119,134],"daily":[41,135],"activities,":[42],"cognitive":[43],"biases":[44,67,94],"are":[45],"one":[46],"most":[49],"common":[50],"ingredients.":[51],"particular,":[53],"reporting":[54],"start":[56],"duration":[58],"an":[60],"activity":[61],"after":[62],"its":[63],"finalisation":[64],"may":[65],"incorporate":[66],"introduced":[68],"personal":[70],"time":[71,84],"perceptions,":[72],"as":[73,75],"well":[74],"imprecision":[77],"lack":[79],"granularity":[81],"due":[82],"rounding.":[85],"Here":[86],"we":[87],"propose":[88],"a":[89,116,131],"method":[90,129],"model":[92],"human":[93],"argue":[99],"for":[100,123],"use":[102],"soft":[104,113],"labels.":[105],"Experimental":[106],"results":[107],"synthetic":[109],"data":[110],"show":[111],"that":[112],"labels":[114],"provide":[115],"better":[117],"approximation":[118],"ground":[121],"truth":[122],"several":[124],"metrics.":[125],"We":[126],"showcase":[127],"real":[132],"dataset":[133],"activities.":[136]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-17T09:09:15.849793","created_date":"2025-10-10T00:00:00"}