{"id":"https://openalex.org/W4399563910","doi":"https://doi.org/10.1109/cogsima61085.2024.10553903","title":"Probing the Consistency of Situational Information Extraction with Large Language Models: A Case Study on Crisis Computing","display_name":"Probing the Consistency of Situational Information Extraction with Large Language Models: A Case Study on Crisis Computing","publication_year":2024,"publication_date":"2024-05-07","ids":{"openalex":"https://openalex.org/W4399563910","doi":"https://doi.org/10.1109/cogsima61085.2024.10553903"},"language":"en","primary_location":{"id":"doi:10.1109/cogsima61085.2024.10553903","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cogsima61085.2024.10553903","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Conference on Cognitive and Computational Aspects of Situation Management (CogSIMA)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012675934","display_name":"Andrea Salfinger","orcid":"https://orcid.org/0000-0003-4160-3871"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Andrea Salfinger","raw_affiliation_strings":["University of Udine,Computer Science and Physics,Department of Mathematics,Udine,Italy","Department of Mathematics, Computer Science and Physics, University of Udine, Udine, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Udine,Computer Science and Physics,Department of Mathematics,Udine,Italy","institution_ids":["https://openalex.org/I129043915"]},{"raw_affiliation_string":"Department of Mathematics, Computer Science and Physics, University of Udine, Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046926620","display_name":"Lauro Snidaro","orcid":"https://orcid.org/0000-0003-3828-9017"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Lauro Snidaro","raw_affiliation_strings":["University of Udine,Computer Science and Physics,Department of Mathematics,Udine,Italy","Department of Mathematics, Computer Science and Physics, University of Udine, Udine, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Udine,Computer Science and Physics,Department of Mathematics,Udine,Italy","institution_ids":["https://openalex.org/I129043915"]},{"raw_affiliation_string":"Department of Mathematics, Computer Science and Physics, University of Udine, Udine, Italy","institution_ids":["https://openalex.org/I129043915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.3156,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.91282874,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"91","last_page":"98"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11121","display_name":"Public Relations and Crisis Communication","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11121","display_name":"Public Relations and Crisis Communication","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/3315","display_name":"Communication"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9505000114440918,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9351000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.6851034164428711},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.669441819190979},{"id":"https://openalex.org/keywords/information-extraction","display_name":"Information extraction","score":0.5113810896873474},{"id":"https://openalex.org/keywords/situational-ethics","display_name":"Situational ethics","score":0.5054666996002197},{"id":"https://openalex.org/keywords/situation-awareness","display_name":"Situation awareness","score":0.45977815985679626},{"id":"https://openalex.org/keywords/extraction","display_name":"Extraction (chemistry)","score":0.415052592754364},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.34865057468414307},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3256187438964844},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.30239975452423096},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1553100347518921},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10222160816192627},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.08958345651626587}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.6851034164428711},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.669441819190979},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.5113810896873474},{"id":"https://openalex.org/C9114305","wikidata":"https://www.wikidata.org/wiki/Q1428317","display_name":"Situational ethics","level":2,"score":0.5054666996002197},{"id":"https://openalex.org/C145804949","wikidata":"https://www.wikidata.org/wiki/Q478123","display_name":"Situation awareness","level":2,"score":0.45977815985679626},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.415052592754364},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34865057468414307},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3256187438964844},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30239975452423096},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1553100347518921},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10222160816192627},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.08958345651626587},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/cogsima61085.2024.10553903","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cogsima61085.2024.10553903","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE Conference on Cognitive and Computational Aspects of Situation Management (CogSIMA)","raw_type":"proceedings-article"},{"id":"pmh:oai:air.uniud.it:11390/1279004","is_oa":false,"landing_page_url":"https://hdl.handle.net/11390/1279004","pdf_url":null,"source":{"id":"https://openalex.org/S4306401163","display_name":"Institutional Research Information System (University of Udine)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I129043915","host_organization_name":"University of Udine","host_organization_lineage":["https://openalex.org/I129043915"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321181","display_name":"Austrian Science Fund","ror":"https://ror.org/013tf3c58"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1564417220","https://openalex.org/W2252616656","https://openalex.org/W2440379835","https://openalex.org/W2979826702","https://openalex.org/W3133702157","https://openalex.org/W4281481998","https://openalex.org/W4285178342","https://openalex.org/W4292779060","https://openalex.org/W4309674289","https://openalex.org/W4385245566","https://openalex.org/W4386372206","https://openalex.org/W6631217535","https://openalex.org/W6739901393","https://openalex.org/W6778883912","https://openalex.org/W6838589991","https://openalex.org/W6856649133","https://openalex.org/W6856966639"],"related_works":["https://openalex.org/W2012350746","https://openalex.org/W3024912289","https://openalex.org/W2415747217","https://openalex.org/W4389264631","https://openalex.org/W2561023719","https://openalex.org/W4382644910","https://openalex.org/W4386286863","https://openalex.org/W2094708502","https://openalex.org/W2143767096","https://openalex.org/W4381733965"],"abstract_inverted_index":{"The":[0],"recently":[1],"introduced":[2],"foundation":[3],"models":[4],"for":[5,127,159,192],"language":[6,25],"modeling,":[7],"also":[8,29],"known":[9,42],"as":[10,43],"Large":[11],"Language":[12],"Models":[13],"(LLMs),":[14],"have":[15],"demonstrated":[16],"breakthrough":[17],"capabilities":[18,118],"on":[19,96,119],"text":[20],"summarization":[21,117],"and":[22,45,70,80,116,181,195],"contextualized":[23],"natural":[24],"processing.":[26],"However,":[27],"these":[28],"suffer":[30],"from":[31,100,104,132],"inherent":[32],"deficiencies":[33],"like":[34],"the":[35,55,67,105,120,150,172],"occasional":[36],"generation":[37],"of":[38,49,58,123,149,174],"factually":[39],"wrong":[40],"information,":[41],"hallucinations,":[44],"a":[46,97,156,189,193],"weak":[47],"consistency":[48,173],"produced":[50],"answers":[51],"strongly":[52],"varying":[53],"with":[54],"exact":[56],"phrasing":[57],"their":[59],"input":[60,89],"query,":[61],"i.e.,":[62],"prompt.":[63],"Hence,":[64],"this":[65,94,153,167],"raises":[66],"question":[68,95],"whether":[69],"how":[71],"LLMs":[72],"could":[73],"replace":[74],"or":[75],"complement":[76],"traditional":[77],"information":[78,84,125,163,176,197],"extraction":[79,164,198],"fusion":[81,85],"modules":[82],"in":[83,142],"pipelines":[86],"involving":[87],"textual":[88],"sources.":[90],"We":[91],"empirically":[92],"examine":[93],"case":[98],"study":[99],"crisis":[101,129],"computing,":[102],"taken":[103],"established":[106],"CrisisFacts":[107],"benchmark":[108],"dataset,":[109],"by":[110],"probing":[111],"an":[112],"LLM\u2019s":[113],"situation":[114,130],"understanding":[115],"target":[121],"task":[122],"extracting":[124],"relevant":[126],"establishing":[128],"awareness":[131],"social":[133,137],"media":[134,138],"corpora.":[135],"Since":[136],"messages":[139],"are":[140],"exchanged":[141],"real-time,":[143],"typically":[144],"targeting":[145],"human":[146],"readers":[147],"aware":[148],"situational":[151,162],"context,":[152],"domain":[154],"represents":[155,188],"prime":[157],"testbed":[158],"evaluating":[160],"LLMs\u2019":[161],"capabilities.":[165],"In":[166],"work,":[168],"we":[169],"specifically":[170],"investigate":[171],"extracted":[175],"across":[177],"different":[178,182],"model":[179],"configurations":[180],"but":[183],"semantically":[184],"similar":[185],"prompts,":[186],"which":[187],"crucial":[190],"prerequisite":[191],"reliable":[194],"trustworthy":[196],"component.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}