{"id":"https://openalex.org/W4385436377","doi":"https://doi.org/10.1145/3586183.3606776","title":"PEANUT: A Human-AI Collaborative Tool for Annotating Audio-Visual Data","display_name":"PEANUT: A Human-AI Collaborative Tool for Annotating Audio-Visual Data","publication_year":2023,"publication_date":"2023-10-20","ids":{"openalex":"https://openalex.org/W4385436377","doi":"https://doi.org/10.1145/3586183.3606776"},"language":"en","primary_location":{"id":"doi:10.1145/3586183.3606776","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3586183.3606776","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 36th Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2307.15167","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081039383","display_name":"Zheng Zhang","orcid":"https://orcid.org/0000-0002-7040-2326"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zheng Zhang","raw_affiliation_strings":["Department of Computer Science and Engineering, University of Notre Dame, United States and Department of Computer Science and Engineering, University of Notre Dame, United States"],"raw_orcid":"https://orcid.org/0000-0002-7040-2326","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of Notre Dame, United States and Department of Computer Science and Engineering, University of Notre Dame, United States","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074125965","display_name":"Zheng Ning","orcid":"https://orcid.org/0000-0001-7374-7453"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zheng Ning","raw_affiliation_strings":["Department of Computer Science and Engineering, University of Notre Dame, United States and Department of Computer Science and Engineering, University of Notre Dame, United States"],"raw_orcid":"https://orcid.org/0000-0001-7374-7453","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of Notre Dame, United States and Department of Computer Science and Engineering, University of Notre Dame, United States","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064805926","display_name":"Chenliang Xu","orcid":"https://orcid.org/0000-0002-2183-822X"},"institutions":[{"id":"https://openalex.org/I5388228","display_name":"University of Rochester","ror":"https://ror.org/022kthw22","country_code":"US","type":"education","lineage":["https://openalex.org/I5388228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chenliang Xu","raw_affiliation_strings":["Department of Computer Science, University of Rochester, United States and Department of Computer Science, University of Rochester, United States"],"raw_orcid":"https://orcid.org/0000-0002-2183-822X","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Rochester, United States and Department of Computer Science, University of Rochester, United States","institution_ids":["https://openalex.org/I5388228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101835756","display_name":"Yapeng Tian","orcid":"https://orcid.org/0000-0003-1423-4513"},"institutions":[{"id":"https://openalex.org/I162577319","display_name":"The University of Texas at Dallas","ror":"https://ror.org/049emcs32","country_code":"US","type":"education","lineage":["https://openalex.org/I162577319"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yapeng Tian","raw_affiliation_strings":["Department of Computer Science, University of Texas at Dallas, United States and Department of Computer Science, University of Texas at Dallas, United States"],"raw_orcid":"https://orcid.org/0000-0003-1423-4513","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Texas at Dallas, United States and Department of Computer Science, University of Texas at Dallas, United States","institution_ids":["https://openalex.org/I162577319"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007240808","display_name":"Toby Jia-Jun Li","orcid":"https://orcid.org/0000-0001-7902-7625"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Toby Jia-Jun Li","raw_affiliation_strings":["Department of Computer Science and Engineering, University of Notre Dame, United States and Department of Computer Science and Engineering, University of Notre Dame, United States"],"raw_orcid":"https://orcid.org/0000-0001-7902-7625","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of Notre Dame, United States and Department of Computer Science and Engineering, University of Notre Dame, United States","institution_ids":["https://openalex.org/I107639228"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8700866103172302},{"id":"https://openalex.org/keywords/audio-visual","display_name":"Audio visual","score":0.7364468574523926},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.673782229423523},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6718228459358215},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5508718490600586},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.5502876043319702},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.514094889163971},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.45632269978523254},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.4519380033016205},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4510464370250702},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4414599537849426},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4282163679599762},{"id":"https://openalex.org/keywords/frame","display_name":"Frame (networking)","score":0.42551666498184204},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3808956742286682},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.34665706753730774}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8700866103172302},{"id":"https://openalex.org/C3017588708","wikidata":"https://www.wikidata.org/wiki/Q758901","display_name":"Audio visual","level":2,"score":0.7364468574523926},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.673782229423523},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6718228459358215},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5508718490600586},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.5502876043319702},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.514094889163971},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.45632269978523254},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.4519380033016205},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4510464370250702},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4414599537849426},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4282163679599762},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.42551666498184204},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3808956742286682},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.34665706753730774},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3586183.3606776","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3586183.3606776","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 36th Annual ACM Symposium on User Interface Software and Technology","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2307.15167","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.15167","pdf_url":"https://arxiv.org/pdf/2307.15167","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2307.15167","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.15167","pdf_url":"https://arxiv.org/pdf/2307.15167","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.47999998927116394,"display_name":"Zero hunger","id":"https://metadata.un.org/sdg/2"}],"awards":[{"id":"https://openalex.org/G1887447798","display_name":"Collaborative Research: SHF: Medium: Towards More Human-like AI Models of Source Code","funder_award_id":"2211428","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6322052660","display_name":null,"funder_award_id":"CCF-2211428","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307791","display_name":"Cisco Systems","ror":"https://ror.org/03yt1ez60"},{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385436377.pdf","grobid_xml":"https://content.openalex.org/works/W4385436377.grobid-xml"},"referenced_works_count":94,"referenced_works":["https://openalex.org/W75502933","https://openalex.org/W343636949","https://openalex.org/W1861492603","https://openalex.org/W1933349210","https://openalex.org/W1965463297","https://openalex.org/W1977011589","https://openalex.org/W1993223663","https://openalex.org/W2013489329","https://openalex.org/W2022710553","https://openalex.org/W2044102377","https://openalex.org/W2058888459","https://openalex.org/W2059216172","https://openalex.org/W2073338313","https://openalex.org/W2073408573","https://openalex.org/W2103018059","https://openalex.org/W2105582566","https://openalex.org/W2110151287","https://openalex.org/W2117281325","https://openalex.org/W2129842875","https://openalex.org/W2132667707","https://openalex.org/W2158515176","https://openalex.org/W2160815625","https://openalex.org/W2250225488","https://openalex.org/W2250999640","https://openalex.org/W2560835477","https://openalex.org/W2565639579","https://openalex.org/W2570764145","https://openalex.org/W2610226709","https://openalex.org/W2610474269","https://openalex.org/W2619697695","https://openalex.org/W2624340939","https://openalex.org/W2626312230","https://openalex.org/W2769041395","https://openalex.org/W2889466822","https://openalex.org/W2908965201","https://openalex.org/W2911396267","https://openalex.org/W2916904544","https://openalex.org/W2917458986","https://openalex.org/W2918341242","https://openalex.org/W2931433835","https://openalex.org/W2941766203","https://openalex.org/W2959716049","https://openalex.org/W2962865004","https://openalex.org/W2962960500","https://openalex.org/W2963115079","https://openalex.org/W2963218389","https://openalex.org/W2963680395","https://openalex.org/W2964109005","https://openalex.org/W2964213933","https://openalex.org/W2966284335","https://openalex.org/W2972018816","https://openalex.org/W2975634117","https://openalex.org/W2981630388","https://openalex.org/W2982619606","https://openalex.org/W2982624843","https://openalex.org/W2985144848","https://openalex.org/W2988200020","https://openalex.org/W2990113535","https://openalex.org/W2998732476","https://openalex.org/W2999876407","https://openalex.org/W3031000691","https://openalex.org/W3094211608","https://openalex.org/W3094385957","https://openalex.org/W3094550259","https://openalex.org/W3101577715","https://openalex.org/W3108367559","https://openalex.org/W3118813946","https://openalex.org/W3119940315","https://openalex.org/W3120163087","https://openalex.org/W3124026294","https://openalex.org/W3156222574","https://openalex.org/W3156281513","https://openalex.org/W3158839228","https://openalex.org/W3160237369","https://openalex.org/W3160526894","https://openalex.org/W3162176464","https://openalex.org/W3163419709","https://openalex.org/W3163469193","https://openalex.org/W3169318522","https://openalex.org/W3174057701","https://openalex.org/W3175335326","https://openalex.org/W3197584784","https://openalex.org/W3211248688","https://openalex.org/W4200436929","https://openalex.org/W4206723194","https://openalex.org/W4214701094","https://openalex.org/W4246750028","https://openalex.org/W4288086169","https://openalex.org/W4289665794","https://openalex.org/W4312755049","https://openalex.org/W4319299886","https://openalex.org/W4360991127","https://openalex.org/W4366548345","https://openalex.org/W4385572926"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2271369634","https://openalex.org/W2185469136","https://openalex.org/W4301143707","https://openalex.org/W2952745240"],"abstract_inverted_index":{"Audio-visual":[0],"learning":[1],"seeks":[2],"to":[3,91,96],"enhance":[4],"the":[5,10,13,31,43,76,93,101,120],"computer\u2019s":[6],"multi-modal":[7,77],"perception":[8],"leveraging":[9],"correlation":[11],"between":[12],"auditory":[14],"and":[15,29,33,54,62,83,88,100],"visual":[16],"modalities.":[17],"Despite":[18],"their":[19],"many":[20],"useful":[21],"downstream":[22],"tasks,":[23,82],"such":[24],"as":[25],"video":[26],"retrieval,":[27],"AR/VR,":[28],"accessibility,":[30],"performance":[32],"adoption":[34],"of":[35,45,103],"existing":[36],"audio-visual":[37,49,66,121],"models":[38,90],"have":[39],"been":[40],"impeded":[41],"by":[42],"availability":[44],"high-quality":[46],"datasets.":[47],"Annotating":[48],"datasets":[50],"is":[51],"laborious,":[52],"expensive,":[53],"time-consuming.":[55],"To":[56],"address":[57],"this":[58],"challenge,":[59],"we":[60],"designed":[61],"developed":[63],"an":[64],"efficient":[65],"annotation":[67,123,128],"tool":[68],"called":[69],"Peanut.":[70],"Peanut\u2019s":[71],"human-AI":[72],"collaborative":[73],"pipeline":[74],"separates":[75],"task":[78],"into":[79],"two":[80],"single-modal":[81],"utilizes":[84],"state-of-the-art":[85],"object":[86],"detection":[87],"sound-tagging":[89],"reduce":[92],"annotators\u2019":[94],"effort":[95],"process":[97,124],"each":[98],"frame":[99],"number":[102],"manually-annotated":[104],"frames":[105],"needed.":[106],"A":[107],"within-subject":[108],"user":[109],"study":[110],"with":[111],"20":[112],"participants":[113],"found":[114],"that":[115],"Peanut":[116],"can":[117],"significantly":[118],"accelerate":[119],"data":[122],"while":[125],"maintaining":[126],"high":[127],"accuracy.":[129]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}