{"id":"https://openalex.org/W4402509116","doi":"https://doi.org/10.1109/jbhi.2024.3460533","title":"A Human-in-the-Loop Method for Annotation of Events in Biomedical Signals","display_name":"A Human-in-the-Loop Method for Annotation of Events in Biomedical Signals","publication_year":2024,"publication_date":"2024-09-13","ids":{"openalex":"https://openalex.org/W4402509116","doi":"https://doi.org/10.1109/jbhi.2024.3460533","pmid":"https://pubmed.ncbi.nlm.nih.gov/39269811"},"language":"en","primary_location":{"id":"doi:10.1109/jbhi.2024.3460533","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jbhi.2024.3460533","pdf_url":null,"source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://lirias.kuleuven.be/retrieve/d7d7432b-b01a-407d-9ba1-99b822c56671","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058516663","display_name":"Nick Seeuws","orcid":"https://orcid.org/0000-0002-7024-3417"},"institutions":[{"id":"https://openalex.org/I99464096","display_name":"KU Leuven","ror":"https://ror.org/05f950310","country_code":"BE","type":"education","lineage":["https://openalex.org/I99464096"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Nick Seeuws","raw_affiliation_strings":["Department of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","Dept. of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium"],"raw_orcid":"https://orcid.org/0000-0002-7024-3417","affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","institution_ids":["https://openalex.org/I99464096"]},{"raw_affiliation_string":"Dept. of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","institution_ids":["https://openalex.org/I99464096"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064593698","display_name":"Maarten De Vos","orcid":"https://orcid.org/0000-0002-3482-5145"},"institutions":[{"id":"https://openalex.org/I99464096","display_name":"KU Leuven","ror":"https://ror.org/05f950310","country_code":"BE","type":"education","lineage":["https://openalex.org/I99464096"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Maarten De Vos","raw_affiliation_strings":["Department of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","Dept. of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","institution_ids":["https://openalex.org/I99464096"]},{"raw_affiliation_string":"Dept. of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","institution_ids":["https://openalex.org/I99464096"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065884964","display_name":"Alexander Bertrand","orcid":"https://orcid.org/0000-0002-4827-8568"},"institutions":[{"id":"https://openalex.org/I99464096","display_name":"KU Leuven","ror":"https://ror.org/05f950310","country_code":"BE","type":"education","lineage":["https://openalex.org/I99464096"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Alexander Bertrand","raw_affiliation_strings":["Department of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","Dept. of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium"],"raw_orcid":"https://orcid.org/0000-0002-4827-8568","affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","institution_ids":["https://openalex.org/I99464096"]},{"raw_affiliation_string":"Dept. of Electrical Engineering (ESAT), Stadius Center for Dynamical Systems, Signal Processing and Data Analytics (STADIUS), KU Leuven, Leuven, Belgium","institution_ids":["https://openalex.org/I99464096"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I99464096"],"apc_list":null,"apc_paid":null,"fwci":0.2944,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.61943494,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"29","issue":"1","first_page":"95","last_page":"106"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11021","display_name":"ECG Monitoring and Analysis","score":0.12189999967813492,"subfield":{"id":"https://openalex.org/subfields/2705","display_name":"Cardiology and Cardiovascular Medicine"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11021","display_name":"ECG Monitoring and Analysis","score":0.12189999967813492,"subfield":{"id":"https://openalex.org/subfields/2705","display_name":"Cardiology and Cardiovascular Medicine"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.07940000295639038,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.07020000368356705,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6445397734642029},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.576624870300293},{"id":"https://openalex.org/keywords/human-in-the-loop","display_name":"Human-in-the-loop","score":0.4824874699115753},{"id":"https://openalex.org/keywords/loop","display_name":"Loop (graph theory)","score":0.4609329104423523},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3650962710380554},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14866265654563904}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6445397734642029},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.576624870300293},{"id":"https://openalex.org/C2780626000","wikidata":"https://www.wikidata.org/wiki/Q5936775","display_name":"Human-in-the-loop","level":2,"score":0.4824874699115753},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.4609329104423523},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3650962710380554},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14866265654563904},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012815","descriptor_name":"Signal Processing, Computer-Assisted","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":3,"locations":[{"id":"doi:10.1109/jbhi.2024.3460533","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jbhi.2024.3460533","pdf_url":null,"source":{"id":"https://openalex.org/S2495854775","display_name":"IEEE Journal of Biomedical and Health Informatics","issn_l":"2168-2194","issn":["2168-2194","2168-2208"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal of Biomedical and Health Informatics","raw_type":"journal-article"},{"id":"pmid:39269811","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39269811","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE journal of biomedical and health informatics","raw_type":null},{"id":"pmh:oai:lirias2repo.kuleuven.be:20.500.12942/751199","is_oa":true,"landing_page_url":"https://lirias.kuleuven.be/handle/20.500.12942/751199","pdf_url":"https://lirias.kuleuven.be/retrieve/d7d7432b-b01a-407d-9ba1-99b822c56671","source":{"id":"https://openalex.org/S7407055369","display_name":"Lirias","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Ieee Journal Of Biomedical And Health Informatics, vol. 29 (1), (95-106)","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:lirias2repo.kuleuven.be:20.500.12942/751199","is_oa":true,"landing_page_url":"https://lirias.kuleuven.be/handle/20.500.12942/751199","pdf_url":"https://lirias.kuleuven.be/retrieve/d7d7432b-b01a-407d-9ba1-99b822c56671","source":{"id":"https://openalex.org/S7407055369","display_name":"Lirias","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Ieee Journal Of Biomedical And Health Informatics, vol. 29 (1), (95-106)","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[{"score":0.5400000214576721,"id":"https://metadata.un.org/sdg/2","display_name":"Zero hunger"}],"awards":[{"id":"https://openalex.org/G3159641495","display_name":null,"funder_award_id":"802895","funder_id":"https://openalex.org/F4320334678","funder_display_name":"European Research Council"},{"id":"https://openalex.org/G434337789","display_name":null,"funder_award_id":"HORIZON2020","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5191705454","display_name":null,"funder_award_id":"802895","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321730","display_name":"Fonds Wetenschappelijk Onderzoek","ror":"https://ror.org/03qtxy027"},{"id":"https://openalex.org/F4320322308","display_name":"KU Leuven","ror":"https://ror.org/05f950310"},{"id":"https://openalex.org/F4320327336","display_name":"Vlaamse regering","ror":null},{"id":"https://openalex.org/F4320334678","display_name":"European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402509116.pdf","grobid_xml":"https://content.openalex.org/works/W4402509116.grobid-xml"},"referenced_works_count":49,"referenced_works":["https://openalex.org/W1969184353","https://openalex.org/W1984207165","https://openalex.org/W2021044602","https://openalex.org/W2038436648","https://openalex.org/W2085413319","https://openalex.org/W2087660979","https://openalex.org/W2101709445","https://openalex.org/W2122930900","https://openalex.org/W2201011793","https://openalex.org/W2316075464","https://openalex.org/W2758219826","https://openalex.org/W2793841943","https://openalex.org/W2794363845","https://openalex.org/W2794550444","https://openalex.org/W2795691199","https://openalex.org/W2809183397","https://openalex.org/W2888358068","https://openalex.org/W2893892260","https://openalex.org/W2912461631","https://openalex.org/W2915893085","https://openalex.org/W2917266614","https://openalex.org/W2963290013","https://openalex.org/W2963939953","https://openalex.org/W2968959539","https://openalex.org/W2969925260","https://openalex.org/W2971225399","https://openalex.org/W2984353870","https://openalex.org/W3012312688","https://openalex.org/W3033527224","https://openalex.org/W3042619474","https://openalex.org/W3118262189","https://openalex.org/W3118932422","https://openalex.org/W3130192038","https://openalex.org/W3130855197","https://openalex.org/W3131318685","https://openalex.org/W3131830830","https://openalex.org/W3133431217","https://openalex.org/W3153885474","https://openalex.org/W3165735694","https://openalex.org/W3197046559","https://openalex.org/W3203816956","https://openalex.org/W4212772483","https://openalex.org/W4291001109","https://openalex.org/W4296707141","https://openalex.org/W4312443924","https://openalex.org/W4317698013","https://openalex.org/W4390494296","https://openalex.org/W4392666369","https://openalex.org/W6756615331"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328","https://openalex.org/W4367173559"],"abstract_inverted_index":{"OBJECTIVE:":[0],"Building":[1],"large-scale":[2],"data":[3,18,96,152],"bases":[4],"of":[5,25,39,59,91,116,126,131,156,166,216],"biomedical":[6,189,217],"signal":[7,190,218],"recordings":[8,35],"for":[9,186],"training":[10],"artificial-intelligence":[11],"systems":[12],"involves":[13],"substantial":[14,124,209],"human":[15,51,79,133,175],"effort":[16,178],"in":[17,52,150,158,188,211],"processing":[19],"and":[20,36,104,142,154,176,197,214],"annotation.":[21,219],"In":[22],"the":[23,34,53,57,60,88,94,108,132,143,161,167,183,212],"case":[24],"event":[26,71,117],"detection,":[27],"experts":[28],"need":[29],"to":[30,55,82,106,111,194],"exhaustively":[31],"scroll":[32],"through":[33,93],"highlight":[37],"events":[38,67,85,127,149,157,187],"interest.":[40],"METHODS:":[41],"We":[42],"propose":[43],"an":[44,70],"iterative":[45],"annotation":[46,61,184],"support":[47],"algorithm":[48,64,99,121,204],"with":[49,138,160],"a":[50,113,123,129,139],"loop":[54],"improve":[56],"efficiency":[58,215],"process.":[62],"Our":[63,98,120,170],"generates":[65],"proposal":[66,102],"based":[68],"on":[69,75,200],"detection":[72],"model":[73],"trained":[74],"incomplete":[76],"annotations.":[77,118],"The":[78],"only":[80,163],"needs":[81],"verify":[83],"candidate":[84],"proposed":[86],"by":[87],"tool":[89],"instead":[90],"scrolling":[92],"entire":[95],"set.":[97],"iterates":[100],"between":[101],"generation":[103],"verification":[105],"leverage":[107],"human-in-the-loop":[109,162],"feedback":[110],"obtain":[112],"growing":[114],"set":[115,153],"RESULTS:":[119],"finds":[122],"amount":[125],"at":[128],"fraction":[130],"time":[134],"spent":[135],"when":[136],"comparing":[137],"benchmark":[140],"method":[141],"normal":[144],"manual":[145],"process,":[146],"finding":[147],"all":[148],"one":[151],"70%":[155],"another":[159],"viewing":[164],"20%":[165],"data.":[168],"CONCLUSION:":[169],"results":[171],"show":[172],"that":[173],"combining":[174],"computer":[177],"can":[179],"substantially":[180],"speed":[181],"up":[182],"process":[185],"processing.":[191],"SIGNIFICANCE:":[192],"Due":[193],"its":[195],"simplicity":[196],"minimal":[198],"reliance":[199],"task-specific":[201],"information,":[202],"our":[203],"is":[205],"broadly":[206],"applicable,":[207],"unlocking":[208],"improvements":[210],"scalability":[213]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-08-06T08:24:18.245995","created_date":"2025-10-10T00:00:00"}