{"id":"https://openalex.org/W4392294975","doi":"https://doi.org/10.1145/3649457","title":"Man and the Machine: Effects of AI-assisted Human Labeling on Interactive Annotation of Real-time Video Streams","display_name":"Man and the Machine: Effects of AI-assisted Human Labeling on Interactive Annotation of Real-time Video Streams","publication_year":2024,"publication_date":"2024-02-29","ids":{"openalex":"https://openalex.org/W4392294975","doi":"https://doi.org/10.1145/3649457"},"language":"en","primary_location":{"id":"doi:10.1145/3649457","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3649457","pdf_url":null,"source":{"id":"https://openalex.org/S4210173818","display_name":"ACM Transactions on Interactive Intelligent Systems","issn_l":"2160-6455","issn":["2160-6455","2160-6463"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Interactive Intelligent Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013051223","display_name":"Marko Radeta","orcid":"https://orcid.org/0000-0002-4842-3622"},"institutions":[{"id":"https://openalex.org/I4068193","display_name":"University of Belgrade","ror":"https://ror.org/02qsmb048","country_code":"RS","type":"education","lineage":["https://openalex.org/I4068193"]},{"id":"https://openalex.org/I58028834","display_name":"Universidade da Madeira","ror":"https://ror.org/0442zbe52","country_code":"PT","type":"education","lineage":["https://openalex.org/I58028834"]}],"countries":["PT","RS"],"is_corresponding":false,"raw_author_name":"Marko Radeta","raw_affiliation_strings":["Wave Labs, MARE/ARNET/ARDITI, University of Madeira, Funchal, Portugal, University of Belgrade, Funchal, Belgrade, Serbia"],"raw_orcid":"https://orcid.org/0000-0002-4842-3622","affiliations":[{"raw_affiliation_string":"Wave Labs, MARE/ARNET/ARDITI, University of Madeira, Funchal, Portugal, University of Belgrade, Funchal, Belgrade, Serbia","institution_ids":["https://openalex.org/I58028834","https://openalex.org/I4068193"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065292621","display_name":"R\u00faben Freitas","orcid":"https://orcid.org/0000-0002-0378-108X"},"institutions":[{"id":"https://openalex.org/I58028834","display_name":"Universidade da Madeira","ror":"https://ror.org/0442zbe52","country_code":"PT","type":"education","lineage":["https://openalex.org/I58028834"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Ruben Freitas","raw_affiliation_strings":["Wave Labs, MARE/ARNET/ARDITI, University of Madeira, Funchal, Portugal"],"raw_orcid":"https://orcid.org/0000-0002-0378-108X","affiliations":[{"raw_affiliation_string":"Wave Labs, MARE/ARNET/ARDITI, University of Madeira, Funchal, Portugal","institution_ids":["https://openalex.org/I58028834"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102973394","display_name":"Claudio Rodrigues","orcid":"https://orcid.org/0000-0001-5510-8527"},"institutions":[{"id":"https://openalex.org/I58028834","display_name":"Universidade da Madeira","ror":"https://ror.org/0442zbe52","country_code":"PT","type":"education","lineage":["https://openalex.org/I58028834"]}],"countries":["PT"],"is_corresponding":false,"raw_author_name":"Claudio Rodrigues","raw_affiliation_strings":["Wave Labs, MARE/ARNET/ARDITI, University of Madeira, Funchal, Portugal"],"raw_orcid":"https://orcid.org/0000-0001-5510-8527","affiliations":[{"raw_affiliation_string":"Wave Labs, MARE/ARNET/ARDITI, University of Madeira, Funchal, Portugal","institution_ids":["https://openalex.org/I58028834"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054275085","display_name":"Agustin Zuniga","orcid":"https://orcid.org/0000-0002-6481-3559"},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Agustin Zuniga","raw_affiliation_strings":["Department of Computer Science, University of Helsinki, Helsinki, Finland"],"raw_orcid":"https://orcid.org/0000-0002-6481-3559","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Helsinki, Helsinki, Finland","institution_ids":["https://openalex.org/I133731052"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033928651","display_name":"Ngoc Thi Nguyen","orcid":"https://orcid.org/0000-0003-1011-8937"},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Ngoc Thi Nguyen","raw_affiliation_strings":["Department of Computer Science, University of Helsinki, Helsinki, Finland"],"raw_orcid":"https://orcid.org/0000-0003-1011-8937","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Helsinki, Helsinki, Finland","institution_ids":["https://openalex.org/I133731052"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080043050","display_name":"Huber Flores","orcid":"https://orcid.org/0000-0003-4551-629X"},"institutions":[{"id":"https://openalex.org/I56085075","display_name":"University of Tartu","ror":"https://ror.org/03z77qz90","country_code":"EE","type":"education","lineage":["https://openalex.org/I56085075"]}],"countries":["EE"],"is_corresponding":false,"raw_author_name":"Huber Flores","raw_affiliation_strings":["Institute of Computer Science, University of Tartu, Tartu, Estonia"],"raw_orcid":"https://orcid.org/0000-0003-4551-629X","affiliations":[{"raw_affiliation_string":"Institute of Computer Science, University of Tartu, Tartu, Estonia","institution_ids":["https://openalex.org/I56085075"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066542585","display_name":"Petteri Nurmi","orcid":"https://orcid.org/0000-0001-8262-6434"},"institutions":[{"id":"https://openalex.org/I133731052","display_name":"University of Helsinki","ror":"https://ror.org/040af2s02","country_code":"FI","type":"education","lineage":["https://openalex.org/I133731052"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Petteri Nurmi","raw_affiliation_strings":["Department of Computer Science, University of Helsinki, Helsinki, Finland"],"raw_orcid":"https://orcid.org/0000-0001-8262-6434","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Helsinki, Helsinki, Finland","institution_ids":["https://openalex.org/I133731052"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.1114,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.79665942,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":"14","issue":"2","first_page":"1","last_page":"22"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.9245601892471313},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8649084568023682},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6438472867012024},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5625942945480347}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.9245601892471313},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8649084568023682},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6438472867012024},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5625942945480347}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3649457","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3649457","pdf_url":null,"source":{"id":"https://openalex.org/S4210173818","display_name":"ACM Transactions on Interactive Intelligent Systems","issn_l":"2160-6455","issn":["2160-6455","2160-6463"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Interactive Intelligent Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2824696408","display_name":"Aquatic Research Infrastructure Network","funder_award_id":"LA/P/0069/2020","funder_id":"https://openalex.org/F4320334779","funder_display_name":"Funda\u00e7\u00e3o para a Ci\u00eancia e a Tecnologia"},{"id":"https://openalex.org/G3484206579","display_name":null,"funder_award_id":"LA/P/0069/2020","funder_id":"https://openalex.org/F4320330295","funder_display_name":"Centro de Ci\u00eancias do Mar e do Ambiente"},{"id":"https://openalex.org/G3860653406","display_name":"Marine and Environmental Sciences Centre","funder_award_id":"UIDB/04292/2020","funder_id":"https://openalex.org/F4320334779","funder_display_name":"Funda\u00e7\u00e3o para a Ci\u00eancia e a Tecnologia"},{"id":"https://openalex.org/G4038499987","display_name":null,"funder_award_id":"UIDB/04292/2020","funder_id":"https://openalex.org/F4320330295","funder_display_name":"Centro de Ci\u00eancias do Mar e do Ambiente"},{"id":"https://openalex.org/G6219495610","display_name":null,"funder_award_id":"339614","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G7786067674","display_name":null,"funder_award_id":"101093865","funder_id":"https://openalex.org/F4320334322","funder_display_name":"HORIZON EUROPE Framework Programme"},{"id":"https://openalex.org/G8134774952","display_name":null,"funder_award_id":"20220138","funder_id":"https://openalex.org/F4320322426","funder_display_name":"Nokia Foundation"},{"id":"https://openalex.org/G8184908717","display_name":"Foundations of Pervasive Sensing Systems","funder_award_id":"339614","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"}],"funders":[{"id":"https://openalex.org/F4320321108","display_name":"Academy of Finland","ror":"https://ror.org/05k73zm37"},{"id":"https://openalex.org/F4320322426","display_name":"Nokia Foundation","ror":"https://ror.org/0401nzk46"},{"id":"https://openalex.org/F4320330295","display_name":"Centro de Ci\u00eancias do Mar e do Ambiente","ror":null},{"id":"https://openalex.org/F4320334322","display_name":"HORIZON EUROPE Framework Programme","ror":null},{"id":"https://openalex.org/F4320334779","display_name":"Funda\u00e7\u00e3o para a Ci\u00eancia e a Tecnologia","ror":"https://ror.org/00snfqn58"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":63,"referenced_works":["https://openalex.org/W1497328373","https://openalex.org/W1666942233","https://openalex.org/W1969161726","https://openalex.org/W1969670714","https://openalex.org/W1974736281","https://openalex.org/W2025208258","https://openalex.org/W2059216172","https://openalex.org/W2064917021","https://openalex.org/W2066032844","https://openalex.org/W2083290278","https://openalex.org/W2084110111","https://openalex.org/W2090847750","https://openalex.org/W2112796928","https://openalex.org/W2120855160","https://openalex.org/W2125943921","https://openalex.org/W2156720790","https://openalex.org/W2162409443","https://openalex.org/W2165698076","https://openalex.org/W2300445845","https://openalex.org/W2507537540","https://openalex.org/W2595991490","https://openalex.org/W2604272474","https://openalex.org/W2610332124","https://openalex.org/W2662969263","https://openalex.org/W2750714283","https://openalex.org/W2753079413","https://openalex.org/W2783539613","https://openalex.org/W2784025535","https://openalex.org/W2792349480","https://openalex.org/W2796096089","https://openalex.org/W2896967904","https://openalex.org/W2901631040","https://openalex.org/W2903142248","https://openalex.org/W2917794201","https://openalex.org/W2920402024","https://openalex.org/W2953522645","https://openalex.org/W2969454448","https://openalex.org/W2972464441","https://openalex.org/W2981972129","https://openalex.org/W3006086117","https://openalex.org/W3047397039","https://openalex.org/W3084169874","https://openalex.org/W3097096317","https://openalex.org/W3156222574","https://openalex.org/W3156281513","https://openalex.org/W3158839228","https://openalex.org/W3176125085","https://openalex.org/W3184711711","https://openalex.org/W3192495122","https://openalex.org/W3203816956","https://openalex.org/W4205354301","https://openalex.org/W4206109607","https://openalex.org/W4210679328","https://openalex.org/W4229449501","https://openalex.org/W4236072661","https://openalex.org/W4236965008","https://openalex.org/W4240477012","https://openalex.org/W4246999471","https://openalex.org/W4249726634","https://openalex.org/W4281849368","https://openalex.org/W4292289324","https://openalex.org/W4297812480","https://openalex.org/W4366003124"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"AI-assisted":[0,18,104,195,257],"interactive":[1,19,88,157],"annotation":[2,20,69,105,146,181,259],"is":[3,29,70],"a":[4,166,173],"powerful":[5],"way":[6],"to":[7,72,185,217],"facilitate":[8,67],"data":[9,135],"annotation\u2014a":[10],"prerequisite":[11],"for":[12,76,199,203,253],"constructing":[13],"robust":[14],"AI":[15,59,77,117,130,163,187,225,229,241],"models.":[16],"While":[17],"has":[21],"been":[22],"extensively":[23],"studied":[24],"in":[25,34,63,87,106,136,234,246,260],"static":[26],"settings,":[27],"less":[28],"known":[30],"about":[31],"its":[32],"usage":[33],"dynamic":[35,107,262],"scenarios":[36],"where":[37,205],"the":[38,98,114,121,127,186,192,224,235,240,247,254],"annotators":[39,62,119,171],"operate":[40],"under":[41],"time":[42],"and":[43,66,101,120,153,155,161,172,179,202,230,244],"cognitive":[44],"constraints,":[45],"e.g.,":[46],"while":[47],"detecting":[48],"suspicious":[49],"or":[50,210,219],"dangerous":[51],"activities":[52],"from":[53],"real-time":[54,137,261],"surveillance":[55],"feeds.":[56],"Understanding":[57],"how":[58],"can":[60],"assist":[61],"these":[64,81],"tasks":[65],"consistent":[68],"paramount":[71],"ensure":[73],"high":[74],"performance":[75,128,147,222,238],"models":[78,131,196],"trained":[79,132],"on":[80,118,126,133],"data.":[82],"We":[83,111,140,250],"address":[84,112],"this":[85],"gap":[86],"machine":[89],"learning":[90],"(IML)":[91],"research,":[92],"contributing":[93],"an":[94],"extensive":[95,142],"investigation":[96],"of":[97,103,116,123,129,194,256],"benefits,":[99],"limitations,":[100],"challenges":[102],"application":[108],"use":[109,255,263],"cases.":[110,264],"both":[113],"effects":[115,122],"(AI)":[124],"annotations":[125,232],"annotated":[134],"video":[138],"annotations.":[139,249],"conduct":[141],"experiments":[143],"that":[144,191],"compare":[145],"at":[148],"two":[149,156],"annotator":[150],"levels":[151],"(expert":[152],"non-expert)":[154],"labeling":[158],"techniques":[159],"(with":[160],"without":[162],"assistance).":[164],"In":[165],"controlled":[167],"study":[168,175],"with":[169,176],"\\(N=34\\)":[170],"follow-up":[174],"51,963":[177],"images":[178],"their":[180],"labels":[182],"being":[183],"input":[184],"model,":[188],"we":[189],"demonstrate":[190],"benefits":[193],"are":[197,207],"greatest":[198],"non-expert":[200],"users":[201,215],"cases":[204],"targets":[206],"only":[208],"partially":[209],"briefly":[211],"visible.":[212],"The":[213],"expert":[214,231,248],"tend":[216],"outperform":[218],"achieve":[220],"similar":[221],"as":[223,239],"model.":[226],"Labels":[227],"combining":[228],"result":[233],"best":[236],"overall":[237],"reduces":[242],"overflow":[243],"latency":[245],"derive":[251],"guidelines":[252],"human":[258]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}