{"id":"https://openalex.org/W2166345396","doi":"https://doi.org/10.1145/2557500.2557512","title":"Toward crowdsourcing micro-level behavior annotations","display_name":"Toward crowdsourcing micro-level behavior annotations","publication_year":2014,"publication_date":"2014-02-18","ids":{"openalex":"https://openalex.org/W2166345396","doi":"https://doi.org/10.1145/2557500.2557512","mag":"2166345396"},"language":"en","primary_location":{"id":"doi:10.1145/2557500.2557512","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2557500.2557512","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 19th international conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102806371","display_name":"Sunghyun Park","orcid":"https://orcid.org/0009-0006-1056-2826"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Sunghyun Park","raw_affiliation_strings":["University of Southern California, Los Angeles, CA, USA"],"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042154617","display_name":"Philippa Shoemark","orcid":"https://orcid.org/0000-0003-4750-5447"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Philippa Shoemark","raw_affiliation_strings":["University of Southern California, Los Angeles, CA, USA"],"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081398601","display_name":"Louis\u2013Philippe Morency","orcid":"https://orcid.org/0000-0001-6376-7696"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Louis-Philippe Morency","raw_affiliation_strings":["University of Southern California, Los Angeles, CA, USA"],"affiliations":[{"raw_affiliation_string":"University of Southern California, Los Angeles, CA, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5102806371"],"corresponding_institution_ids":["https://openalex.org/I1174212"],"apc_list":null,"apc_paid":null,"fwci":4.5122,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.94719174,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"37","last_page":"46"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9850848913192749},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8075416088104248},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7881379127502441},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5988397002220154},{"id":"https://openalex.org/keywords/crowdsourcing-software-development","display_name":"Crowdsourcing software development","score":0.5946452617645264},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5339198708534241},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.5020875930786133},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.449073851108551},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.4153693616390228},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.38886526226997375},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3791276216506958},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3789684772491455},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.37487828731536865},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.36212772130966187},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32556140422821045},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.07359579205513}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9850848913192749},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8075416088104248},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7881379127502441},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5988397002220154},{"id":"https://openalex.org/C32810761","wikidata":"https://www.wikidata.org/wiki/Q17082146","display_name":"Crowdsourcing software development","level":5,"score":0.5946452617645264},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5339198708534241},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5020875930786133},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.449073851108551},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.4153693616390228},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.38886526226997375},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3791276216506958},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3789684772491455},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.37487828731536865},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.36212772130966187},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32556140422821045},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.07359579205513},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2557500.2557512","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2557500.2557512","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 19th international conference on Intelligent User Interfaces","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.6800000071525574}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332236","display_name":"Annenberg School for Communication and Journalism, University of Southern California","ror":"https://ror.org/03taz7m60"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W85391330","https://openalex.org/W188300810","https://openalex.org/W1487796455","https://openalex.org/W1514923623","https://openalex.org/W1970381522","https://openalex.org/W1973200191","https://openalex.org/W2003497265","https://openalex.org/W2035818959","https://openalex.org/W2036625304","https://openalex.org/W2061863227","https://openalex.org/W2099864214","https://openalex.org/W2104749423","https://openalex.org/W2106996050","https://openalex.org/W2114269021","https://openalex.org/W2115915304","https://openalex.org/W2116251929","https://openalex.org/W2119775030","https://openalex.org/W2125943921","https://openalex.org/W2129013408","https://openalex.org/W2135776491","https://openalex.org/W2147069236","https://openalex.org/W2149816711","https://openalex.org/W2153222072","https://openalex.org/W2158880898","https://openalex.org/W2166145477","https://openalex.org/W2170538746","https://openalex.org/W2170989440","https://openalex.org/W2187291759","https://openalex.org/W2264285263","https://openalex.org/W2328893508","https://openalex.org/W2371180841","https://openalex.org/W2382398184","https://openalex.org/W2575536386","https://openalex.org/W3123895079","https://openalex.org/W6666041595"],"related_works":["https://openalex.org/W2512762878","https://openalex.org/W2397281630","https://openalex.org/W2033745466","https://openalex.org/W162737300","https://openalex.org/W2520160102","https://openalex.org/W2520748299","https://openalex.org/W2011363905","https://openalex.org/W2046555014","https://openalex.org/W2548342535","https://openalex.org/W3131060711"],"abstract_inverted_index":{"Research":[0],"that":[1,50,111],"involves":[2],"human":[3],"behavior":[4,15,55,122],"analysis":[5],"usually":[6],"requires":[7],"laborious":[8],"and":[9,53,91],"costly":[10],"efforts":[11,30],"for":[12,42,97,119],"obtaining":[13,120],"micro-level":[14,121],"annotations":[16,56,123],"on":[17],"a":[18,46,71,149],"large":[19],"video":[20,152],"corpus.":[21,153],"With":[22],"the":[23,113,127,142],"emerging":[24],"paradigm":[25],"of":[26,44,67,87,109,115,144],"crowdsourcing":[27,63,117],"however,":[28],"these":[29],"can":[31],"be":[32],"considerably":[33],"reduced.":[34],"We":[35,139],"first":[36],"present":[37,105],"OCTAB":[38],"(Online":[39],"Crowdsourcing":[40],"Tool":[41],"Annotations":[43],"Behaviors),":[45],"web-based":[47],"annotation":[48,131],"tool":[49],"allows":[51],"precise":[52],"convenient":[54],"in":[57,124,130],"videos,":[58,125],"directly":[59],"portable":[60],"to":[61,148],"popular":[62],"platforms.":[64],"As":[65],"part":[66],"OCTAB,":[68],"we":[69,104],"introduce":[70],"training":[72,78,99,135,146],"module":[73],"with":[74],"specialized":[75],"visualizations.":[76],"The":[77],"module's":[79],"design":[80],"was":[81],"inspired":[82],"by":[83],"an":[84,94,106],"observational":[85],"study":[86],"local":[88],"experienced":[89],"coders,":[90],"it":[92],"enables":[93],"iterative":[95],"procedure":[96],"effectively":[98],"crowd":[100,137],"workers":[101],"online.":[102],"Finally,":[103],"extensive":[107],"set":[108],"experiments":[110],"evaluates":[112],"feasibility":[114],"our":[116,145],"approach":[118,147],"showing":[126],"reliability":[128],"improvement":[129],"accuracy":[132],"when":[133],"properly":[134],"online":[136],"workers.":[138],"also":[140],"show":[141],"generalization":[143],"new":[150],"independent":[151]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":7}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}