{"id":"https://openalex.org/W4394000193","doi":"https://doi.org/10.1145/3640543.3645214","title":"Closing the Knowledge Gap in Designing Data Annotation Interfaces for AI-powered Disaster Management Analytic Systems","display_name":"Closing the Knowledge Gap in Designing Data Annotation Interfaces for AI-powered Disaster Management Analytic Systems","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4394000193","doi":"https://doi.org/10.1145/3640543.3645214"},"language":"en","primary_location":{"id":"doi:10.1145/3640543.3645214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645214","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645214","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645214","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076607095","display_name":"Zinat Ara","orcid":null},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Zinat Ara","raw_affiliation_strings":["Information Sciences and Technology, George Mason University, United States"],"raw_orcid":"https://orcid.org/0009-0003-2479-8846","affiliations":[{"raw_affiliation_string":"Information Sciences and Technology, George Mason University, United States","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102958171","display_name":"Hossein Salemi","orcid":"https://orcid.org/0000-0003-3462-9037"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hossein Salemi","raw_affiliation_strings":["Information Sciences and Technology, George Mason University, United States"],"raw_orcid":"https://orcid.org/0000-0003-3462-9037","affiliations":[{"raw_affiliation_string":"Information Sciences and Technology, George Mason University, United States","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059548204","display_name":"Sung-Soo Hong","orcid":"https://orcid.org/0000-0001-6050-5404"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sungsoo Ray Hong","raw_affiliation_strings":["Information Sciences and Technology, George Mason University, United States"],"raw_orcid":"https://orcid.org/0000-0001-6050-5404","affiliations":[{"raw_affiliation_string":"Information Sciences and Technology, George Mason University, United States","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070513669","display_name":"Yasas Senarath","orcid":"https://orcid.org/0000-0002-8759-378X"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yasas Senarath","raw_affiliation_strings":["Information Sciences and Technology, George Mason University, United States"],"raw_orcid":"https://orcid.org/0000-0002-8759-378X","affiliations":[{"raw_affiliation_string":"Information Sciences and Technology, George Mason University, United States","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039056255","display_name":"Steve Peterson","orcid":"https://orcid.org/0000-0001-5429-3855"},"institutions":[{"id":"https://openalex.org/I4210106219","display_name":"Montgomery County Department of Health and Human Services","ror":"https://ror.org/019s1t671","country_code":"US","type":"government","lineage":["https://openalex.org/I4210106219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Steve Peterson","raw_affiliation_strings":["Montgomery County Community Emergency Response Team, United States"],"raw_orcid":"https://orcid.org/0000-0001-5429-3855","affiliations":[{"raw_affiliation_string":"Montgomery County Community Emergency Response Team, United States","institution_ids":["https://openalex.org/I4210106219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060353074","display_name":"Amanda Hughes","orcid":"https://orcid.org/0000-0002-7506-3343"},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Amanda Lee Hughes","raw_affiliation_strings":["Computer Science Department, Brigham Young University, United States"],"raw_orcid":"https://orcid.org/0000-0002-7506-3343","affiliations":[{"raw_affiliation_string":"Computer Science Department, Brigham Young University, United States","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5012073624","display_name":"Hemant Purohit","orcid":"https://orcid.org/0000-0002-4573-8450"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hemant Purohit","raw_affiliation_strings":["Information Sciences and Technology, George Mason University, United States"],"raw_orcid":"https://orcid.org/0000-0002-4573-8450","affiliations":[{"raw_affiliation_string":"Information Sciences and Technology, George Mason University, United States","institution_ids":["https://openalex.org/I162714631"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5076607095"],"corresponding_institution_ids":["https://openalex.org/I162714631"],"apc_list":null,"apc_paid":null,"fwci":3.1237,"has_fulltext":true,"cited_by_count":8,"citation_normalized_percentile":{"value":0.92914144,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"405","last_page":"418"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9825999736785889,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13018","display_name":"Seismology and Earthquake Studies","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10273","display_name":"IoT and Edge/Fog Computing","score":0.9749000072479248,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closing","display_name":"Closing (real estate)","score":0.8189307451248169},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.716586172580719},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5892626047134399},{"id":"https://openalex.org/keywords/emergency-management","display_name":"Emergency management","score":0.4833061695098877},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.4307730197906494},{"id":"https://openalex.org/keywords/data-management","display_name":"Data management","score":0.4293499290943146},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.333232581615448},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.32117384672164917},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2857634425163269},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.18039047718048096}],"concepts":[{"id":"https://openalex.org/C2778775528","wikidata":"https://www.wikidata.org/wiki/Q5135432","display_name":"Closing (real estate)","level":2,"score":0.8189307451248169},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.716586172580719},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5892626047134399},{"id":"https://openalex.org/C62555980","wikidata":"https://www.wikidata.org/wiki/Q1460420","display_name":"Emergency management","level":2,"score":0.4833061695098877},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4307730197906494},{"id":"https://openalex.org/C1668388","wikidata":"https://www.wikidata.org/wiki/Q1149776","display_name":"Data management","level":2,"score":0.4293499290943146},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.333232581615448},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32117384672164917},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2857634425163269},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.18039047718048096},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3640543.3645214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645214","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645214","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3640543.3645214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645214","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645214","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/13","score":0.8100000023841858,"display_name":"Climate action"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320307285","display_name":"Impact Fund","ror":"https://ror.org/00jb20j87"},{"id":"https://openalex.org/F4320309029","display_name":"George Mason University","ror":"https://ror.org/02jqj7156"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394000193.pdf","grobid_xml":"https://content.openalex.org/works/W4394000193.grobid-xml"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W48379747","https://openalex.org/W1966900440","https://openalex.org/W2058868256","https://openalex.org/W2072500283","https://openalex.org/W2116282804","https://openalex.org/W2161607736","https://openalex.org/W2169942798","https://openalex.org/W2290009368","https://openalex.org/W2588896531","https://openalex.org/W2595576831","https://openalex.org/W2607311634","https://openalex.org/W2795821046","https://openalex.org/W2803437449","https://openalex.org/W2941505948","https://openalex.org/W2942161347","https://openalex.org/W2945554431","https://openalex.org/W2954438306","https://openalex.org/W2983697263","https://openalex.org/W3003928769","https://openalex.org/W3015485802","https://openalex.org/W3019489177","https://openalex.org/W3041011740","https://openalex.org/W3041215734","https://openalex.org/W3105662186","https://openalex.org/W3105859057","https://openalex.org/W3105885889","https://openalex.org/W3122778363","https://openalex.org/W3158839228","https://openalex.org/W3187437888","https://openalex.org/W3198526542","https://openalex.org/W3203878905","https://openalex.org/W3207879356","https://openalex.org/W4206139996","https://openalex.org/W4206952613","https://openalex.org/W4224981752","https://openalex.org/W4225100122","https://openalex.org/W4225915922","https://openalex.org/W4256167997","https://openalex.org/W4283703617","https://openalex.org/W4283799550","https://openalex.org/W4288828483","https://openalex.org/W4308765676","https://openalex.org/W4309619902","https://openalex.org/W4319453786","https://openalex.org/W4366588458","https://openalex.org/W4385572161","https://openalex.org/W4387344850"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2116422677","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2517007886","https://openalex.org/W2383994331","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W1522019333","https://openalex.org/W3121919567"],"abstract_inverted_index":{"Data":[0],"annotation":[1,178],"interfaces":[2,205],"predominantly":[3],"leverage":[4],"ground":[5],"truth":[6],"labels":[7],"to":[8,31,42,49,82,88,97,100],"guide":[9],"annotators":[10,34,91],"toward":[11],"accurate":[12],"responses.":[13,104],"With":[14],"the":[15,68,84,127,131,158,162,209],"growing":[16],"adoption":[17],"of":[18,70,77,111,121,133,161,193],"Artificial":[19],"Intelligence":[20],"(AI)":[21],"in":[22,46,176,191],"domain-specific":[23],"professional":[24],"tasks,":[25],"it":[26],"has":[27],"become":[28],"increasingly":[29],"important":[30],"help":[32,151],"beginning":[33],"identify":[35,153],"how":[36],"their":[37,102],"early-stage":[38],"knowledge":[39,123,210],"can":[40],"lead":[41],"inaccurate":[43],"answers,":[44],"which":[45,147],"turn,":[47],"helps":[48],"ensure":[50],"quality":[51],"annotations":[52],"at":[53,207],"scale.":[54],"To":[55],"investigate":[56],"this":[57],"issue,":[58],"we":[59,142],"conducted":[60],"a":[61,119,144],"formative":[62],"study":[63,167],"involving":[64],"eight":[65],"individuals":[66],"from":[67],"field":[69],"disaster":[71],"management,":[72],"each":[73],"possessing":[74],"varying":[75],"levels":[76],"expertise.":[78],"The":[79,165],"goal":[80],"was":[81],"understand":[83],"prevalent":[85],"factors":[86],"contributing":[87],"disagreements":[89],"among":[90,212],"when":[92],"classifying":[93],"Twitter":[94],"messages":[95],"related":[96],"disasters":[98],"and":[99,115,129,156,181,195],"analyze":[101],"respective":[103],"Our":[105],"analysis":[106],"identified":[107],"two":[108,172],"primary":[109],"causes":[110],"disagreement":[112],"between":[113,188],"expert":[114],"beginner":[116],"annotators:":[117],"1)":[118],"lack":[120],"contextual":[122],"or":[124,135],"uncertainty":[125],"about":[126],"situation,":[128],"2)":[130],"absence":[132],"visual":[134],"supplementary":[136],"cues.":[137],"Based":[138],"on":[139],"these":[140,189],"findings,":[141],"designed":[143],"Context":[145,169],"interface,":[146],"generates":[148],"aids":[149],"that":[150],"beginners":[152],"potential":[154],"mistakes":[155],"provide":[157],"hidden":[159],"context":[160],"presented":[163],"tweet.":[164],"summative":[166],"compares":[168],"design":[170],"with":[171,200],"widely":[173],"used":[174],"designs":[175,190],"data":[177],"UI,":[179],"Highlight":[180],"Reasoning-based":[182],"interfaces.":[183],"We":[184,198],"found":[185],"significant":[186],"differences":[187],"terms":[192],"attitudinal":[194],"behavioral":[196],"data.":[197],"conclude":[199],"implications":[201],"for":[202],"designing":[203],"future":[204],"aiming":[206],"closing":[208],"gap":[211],"annotators.":[213]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}