{"id":"https://openalex.org/W7153792188","doi":"https://doi.org/10.1145/3772363.3798872","title":"Who Fails Where? LLM and Human Error Patterns in Endometriosis Ultrasound Report Extraction","display_name":"Who Fails Where? LLM and Human Error Patterns in Endometriosis Ultrasound Report Extraction","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7153792188","doi":"https://doi.org/10.1145/3772363.3798872"},"language":null,"primary_location":{"id":"doi:10.1145/3772363.3798872","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772363.3798872","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Extended Abstracts of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3772363.3798872","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133465179","display_name":"Haiyi Li","orcid":"https://orcid.org/0009-0004-6914-8457"},"institutions":[{"id":"https://openalex.org/I170239107","display_name":"University of South Australia","ror":"https://ror.org/01p93h210","country_code":"AU","type":"education","lineage":["https://openalex.org/I170239107"]},{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Haiyi Li","raw_affiliation_strings":["School of Computer and Mathematical Sciences, Adelaide University, Adelaide, South Australia, Australia"],"raw_orcid":"https://orcid.org/0009-0004-6914-8457","affiliations":[{"raw_affiliation_string":"School of Computer and Mathematical Sciences, Adelaide University, Adelaide, South Australia, Australia","institution_ids":["https://openalex.org/I5681781","https://openalex.org/I170239107"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133438328","display_name":"Yutong Li","orcid":"https://orcid.org/0009-0002-3680-058X"},"institutions":[{"id":"https://openalex.org/I170239107","display_name":"University of South Australia","ror":"https://ror.org/01p93h210","country_code":"AU","type":"education","lineage":["https://openalex.org/I170239107"]},{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yutong Li","raw_affiliation_strings":["School of Computer and Mathematical Sciences, Adelaide University, Adelaide, South Australia, Australia"],"raw_orcid":"https://orcid.org/0009-0002-3680-058X","affiliations":[{"raw_affiliation_string":"School of Computer and Mathematical Sciences, Adelaide University, Adelaide, South Australia, Australia","institution_ids":["https://openalex.org/I5681781","https://openalex.org/I170239107"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123058625","display_name":"Yiheng Chi","orcid":null},"institutions":[{"id":"https://openalex.org/I170239107","display_name":"University of South Australia","ror":"https://ror.org/01p93h210","country_code":"AU","type":"education","lineage":["https://openalex.org/I170239107"]},{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yiheng Chi","raw_affiliation_strings":["School of Computer and Mathematical Sciences, Adelaide University, Adelaide, South Australia, Australia"],"raw_orcid":"https://orcid.org/0009-0005-0916-2182","affiliations":[{"raw_affiliation_string":"School of Computer and Mathematical Sciences, Adelaide University, Adelaide, South Australia, Australia","institution_ids":["https://openalex.org/I5681781","https://openalex.org/I170239107"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021466877","display_name":"Alison Deslandes","orcid":"https://orcid.org/0000-0001-7094-3950"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Alison Deslandes","raw_affiliation_strings":["Robinson Research Institute, Adelaide University, Adelaide, Australia"],"raw_orcid":"https://orcid.org/0000-0001-7094-3950","affiliations":[{"raw_affiliation_string":"Robinson Research Institute, Adelaide University, Adelaide, Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052955194","display_name":"Mathew Leonardi","orcid":"https://orcid.org/0000-0001-5538-6906"},"institutions":[{"id":"https://openalex.org/I98251732","display_name":"McMaster University","ror":"https://ror.org/02fa3aq29","country_code":"CA","type":"education","lineage":["https://openalex.org/I98251732"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mathew Leonardi","raw_affiliation_strings":["Obstetrics and Gynecology, McMaster University, Hamilton, Ontario, Canada"],"raw_orcid":"https://orcid.org/0000-0001-5538-6906","affiliations":[{"raw_affiliation_string":"Obstetrics and Gynecology, McMaster University, Hamilton, Ontario, Canada","institution_ids":["https://openalex.org/I98251732"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077685071","display_name":"Shay Freger","orcid":"https://orcid.org/0000-0001-5534-3965"},"institutions":[{"id":"https://openalex.org/I98251732","display_name":"McMaster University","ror":"https://ror.org/02fa3aq29","country_code":"CA","type":"education","lineage":["https://openalex.org/I98251732"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Shay Freger","raw_affiliation_strings":["McMaster University, Hamilton, Ontario, Canada"],"raw_orcid":"https://orcid.org/0000-0001-5534-3965","affiliations":[{"raw_affiliation_string":"McMaster University, Hamilton, Ontario, Canada","institution_ids":["https://openalex.org/I98251732"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yuan Zhang","orcid":"https://orcid.org/0000-0002-3484-3140"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yuan Zhang","raw_affiliation_strings":["Robinson Research Institute, Adelaide University, Adelaide, Australia"],"raw_orcid":"https://orcid.org/0000-0002-3484-3140","affiliations":[{"raw_affiliation_string":"Robinson Research Institute, Adelaide University, Adelaide, Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052912720","display_name":"Jodie Avery","orcid":"https://orcid.org/0000-0002-8857-9162"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Jodie Avery","raw_affiliation_strings":["Robinson Research Institute, Adelaide University, Adelaide, SA, Australia"],"raw_orcid":"https://orcid.org/0000-0002-8857-9162","affiliations":[{"raw_affiliation_string":"Robinson Research Institute, Adelaide University, Adelaide, SA, Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133446150","display_name":"Mary Louise Hull","orcid":"https://orcid.org/0000-0003-1813-3971"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Mary Louise Hull","raw_affiliation_strings":["Robinson Research Institute, Adelaide University, Adelaide, Australia"],"raw_orcid":"https://orcid.org/0000-0003-1813-3971","affiliations":[{"raw_affiliation_string":"Robinson Research Institute, Adelaide University, Adelaide, Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036805602","display_name":"Hsiang\u2010Ting Chen","orcid":"https://orcid.org/0000-0003-0873-2698"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Hsiang-Ting Chen","raw_affiliation_strings":["School of Computer Science and Information Technology, Adelaide University, Adelaide, South Australia, Australia"],"raw_orcid":"https://orcid.org/0000-0003-0873-2698","affiliations":[{"raw_affiliation_string":"School of Computer Science and Information Technology, Adelaide University, Adelaide, South Australia, Australia","institution_ids":["https://openalex.org/I5681781"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11894","display_name":"Radiology practices and education","score":0.052799999713897705,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11894","display_name":"Radiology practices and education","score":0.052799999713897705,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.04800000041723251,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10935","display_name":"Endometriosis Research and Treatment","score":0.04349999874830246,"subfield":{"id":"https://openalex.org/subfields/2743","display_name":"Reproductive Medicine"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/endometriosis","display_name":"Endometriosis","score":0.5859000086784363},{"id":"https://openalex.org/keywords/ultrasound","display_name":"Ultrasound","score":0.46720001101493835},{"id":"https://openalex.org/keywords/extraction","display_name":"Extraction (chemistry)","score":0.37709999084472656},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.29260000586509705},{"id":"https://openalex.org/keywords/ultrasonography","display_name":"Ultrasonography","score":0.27970001101493835}],"concepts":[{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.7953000068664551},{"id":"https://openalex.org/C2779522080","wikidata":"https://www.wikidata.org/wiki/Q205764","display_name":"Endometriosis","level":2,"score":0.5859000086784363},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.48739999532699585},{"id":"https://openalex.org/C143753070","wikidata":"https://www.wikidata.org/wiki/Q162564","display_name":"Ultrasound","level":2,"score":0.46720001101493835},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.37709999084472656},{"id":"https://openalex.org/C141071460","wikidata":"https://www.wikidata.org/wiki/Q40821","display_name":"Surgery","level":1,"score":0.30399999022483826},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.29260000586509705},{"id":"https://openalex.org/C529618451","wikidata":"https://www.wikidata.org/wiki/Q234904","display_name":"Ultrasonography","level":2,"score":0.27970001101493835},{"id":"https://openalex.org/C3018824978","wikidata":"https://www.wikidata.org/wiki/Q2894891","display_name":"Error analysis","level":2,"score":0.2786000072956085},{"id":"https://openalex.org/C169806903","wikidata":"https://www.wikidata.org/wiki/Q5937752","display_name":"Human error","level":2,"score":0.26269999146461487}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3772363.3798872","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772363.3798872","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Extended Abstracts of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3772363.3798872","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772363.3798872","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Extended Abstracts of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1487381889","https://openalex.org/W2047846130","https://openalex.org/W2095118384","https://openalex.org/W2237502368","https://openalex.org/W2606833494","https://openalex.org/W2765488845","https://openalex.org/W2768488789","https://openalex.org/W2903272045","https://openalex.org/W2966351171","https://openalex.org/W2970837303","https://openalex.org/W3004483087","https://openalex.org/W3159250634","https://openalex.org/W4384071683","https://openalex.org/W4385573087","https://openalex.org/W4403626237"],"related_works":[],"abstract_inverted_index":{"In":[0],"this":[1],"study,":[2],"we":[3,26],"evaluate":[4],"locally":[5],"deployed":[6],"large":[7],"language":[8],"models":[9],"(LLMs)":[10],"for":[11],"converting":[12],"unstructured":[13],"endometriosis":[14],"transvaginal":[15],"ultrasound":[16],"(eTVUS)":[17],"reports":[18],"into":[19],"structured":[20,69,122],"data.":[21],"Across":[22],"49":[23],"de-identified":[24],"reports,":[25],"compared":[27],"three":[28],"on-premise":[29],"LLMs":[30,58],"(7B/8B":[31],"and":[32,59,90,129],"20B":[33,44],"parameters)":[34],"against":[35],"expert":[36],"human":[37,130],"extraction":[38],"using":[39],"a":[40,114],"185-field":[41],"schema.":[42],"The":[43],"model":[45,105],"achieved":[46],"the":[47,54,65,119],"highest":[48],"mean":[49],"accuracy":[50],"(86.02%),":[51],"substantially":[52],"outperforming":[53],"smaller":[55],"models.":[56],"Crucially,":[57],"humans":[60,76,81],"exhibited":[61],"complementary":[62],"error":[63],"patterns:":[64],"LLM":[66,120],"excelled":[67],"on":[68,85,133],"fields":[70,87,134],"(date":[71],"formatting,":[72],"measurement":[73],"decomposition)":[74],"where":[75],"made":[77],"protocol":[78],"errors,":[79,128],"while":[80],"demonstrated":[82],"superior":[83],"performance":[84],"interpretive":[86],"involving":[88],"negation":[89],"clinical":[91,136],"terminology.":[92],"Targeted":[93],"prompt":[94],"engineering":[95],"yielded":[96],"only":[97],"marginal":[98],"gains,":[99],"indicating":[100],"that":[101],"these":[102],"errors":[103],"reflect":[104],"limitations":[106],"rather":[107],"than":[108],"instruction":[109],"gaps.":[110],"These":[111],"findings":[112],"support":[113],"human-in-the-loop":[115],"workflow":[116],"in":[117],"which":[118],"generates":[121],"drafts,":[123],"automated":[124],"validation":[125],"flags":[126],"rule-verifiable":[127],"review":[131],"focuses":[132],"requiring":[135],"interpretation.":[137]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-04-13T00:00:00"}