{"id":"https://openalex.org/W7134047584","doi":"https://doi.org/10.1145/3772318.3791069","title":"Results-Actionability Gap: Understanding How Practitioners Evaluate LLM Products in the Wild","display_name":"Results-Actionability Gap: Understanding How Practitioners Evaluate LLM Products in the Wild","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7134047584","doi":"https://doi.org/10.1145/3772318.3791069"},"language":"en","primary_location":{"id":"doi:10.1145/3772318.3791069","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3791069","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3772318.3791069","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5122976814","display_name":"Willem; id_orcid 0000-0003-0245-1633 van der Maden","orcid":null},"institutions":[{"id":"https://openalex.org/I83467386","display_name":"IT University of Copenhagen","ror":"https://ror.org/02309jg23","country_code":"DK","type":"education","lineage":["https://openalex.org/I83467386"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Willem van der Maden","raw_affiliation_strings":["HCI & Design Section, IT University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0003-0245-1633","affiliations":[{"raw_affiliation_string":"HCI & Design Section, IT University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I83467386"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101678969","display_name":"Malak Sadek","orcid":"https://orcid.org/0000-0001-8284-890X"},"institutions":[{"id":"https://openalex.org/I4210164862","display_name":"Artificial Intelligence in Medicine (Canada)","ror":"https://ror.org/05p590m36","country_code":"CA","type":"company","lineage":["https://openalex.org/I4210164862"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Malak Sadek","raw_affiliation_strings":["Centre for Human-Inspired Artificial Intelligence (CHIA), Cambridge University, Cambridge, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0001-8284-890X","affiliations":[{"raw_affiliation_string":"Centre for Human-Inspired Artificial Intelligence (CHIA), Cambridge University, Cambridge, United Kingdom","institution_ids":["https://openalex.org/I4210164862"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124861782","display_name":"Ziang Xiao","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ziang Xiao","raw_affiliation_strings":["Computer Science, Johns Hopkins University, Baltimore, Maryland, USA"],"raw_orcid":"https://orcid.org/0000-0003-3368-0180","affiliations":[{"raw_affiliation_string":"Computer Science, Johns Hopkins University, Baltimore, Maryland, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083410044","display_name":"Aske Mottelson","orcid":"https://orcid.org/0000-0003-1827-8513"},"institutions":[{"id":"https://openalex.org/I83467386","display_name":"IT University of Copenhagen","ror":"https://ror.org/02309jg23","country_code":"DK","type":"education","lineage":["https://openalex.org/I83467386"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Aske Mottelson","raw_affiliation_strings":["HCI & Design Section, IT University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0003-1827-8513","affiliations":[{"raw_affiliation_string":"HCI & Design Section, IT University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I83467386"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128241549","display_name":"Q. Vera Liao","orcid":null},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Q. Vera Liao","raw_affiliation_strings":["Computer Science and Engineering, University of Michigan, Ann Arbor, Ann Arbor, Michigan, USA"],"raw_orcid":"https://orcid.org/0000-0003-4543-7196","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering, University of Michigan, Ann Arbor, Ann Arbor, Michigan, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020316102","display_name":"Jichen Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I83467386","display_name":"IT University of Copenhagen","ror":"https://ror.org/02309jg23","country_code":"DK","type":"education","lineage":["https://openalex.org/I83467386"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Jichen Zhu","raw_affiliation_strings":["HCI & Design Section, IT University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0001-6740-4550","affiliations":[{"raw_affiliation_string":"HCI & Design Section, IT University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I83467386"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.21468785,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11024","display_name":"Information Systems Theories and Implementation","score":0.17389999330043793,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11024","display_name":"Information Systems Theories and Implementation","score":0.17389999330043793,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.07779999822378159,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.07450000196695328,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.48980000615119934},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.4442000091075897},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.4147000014781952},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.3458000123500824},{"id":"https://openalex.org/keywords/qualitative-research","display_name":"Qualitative research","score":0.32179999351501465}],"concepts":[{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.5422999858856201},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.48980000615119934},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.4442000091075897},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.4147000014781952},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.39980000257492065},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.3725000023841858},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.3653999865055084},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.3458000123500824},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.33009999990463257},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.32659998536109924},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.32179999351501465},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.29820001125335693},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.29319998621940613},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.2892000079154968},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.25870001316070557},{"id":"https://openalex.org/C5395021","wikidata":"https://www.wikidata.org/wiki/Q1061410","display_name":"Program evaluation","level":2,"score":0.2522999942302704},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.2522999942302704}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3772318.3791069","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3791069","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2604.16304","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2604.16304","pdf_url":"https://arxiv.org/pdf/2604.16304","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:pure.atira.dk:openaire/b89d5d18-9511-4b7b-a004-2b3051205037","is_oa":true,"landing_page_url":"https://pure.itu.dk/portal/da/publications/b89d5d18-9511-4b7b-a004-2b3051205037","pdf_url":"https://pure.itu.dk/ws/files/114617114/chi26-result-actionability-gap-vandermaden.pdf","source":{"id":"https://openalex.org/S4377196680","display_name":"IT University Of Copenhagen (IT University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I83467386","host_organization_name":"IT University of Copenhagen","host_organization_lineage":["https://openalex.org/I83467386"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"van der Maden, W, Sadek, M, Xiao, Z, Mottelson, A, Liao, Q V & Zhu, J 2026, Results-Actionability Gap: Understanding How Practitioners Evaluate LLM Products in the Wild. in Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems. Association for Computing Machinery, New York, NY, USA, pp. 1-17, Conference on Human Factors in Computing Systems, Barcelona, Spain, 13/04/2026. https://doi.org/10.1145/3772318.3791069","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.1145/3772318.3791069","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3791069","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.4064059853553772,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G6365535034","display_name":null,"funder_award_id":"NNF20OC0066119","funder_id":"https://openalex.org/F4320325957","funder_display_name":"Novo Nordisk Fonden"}],"funders":[{"id":"https://openalex.org/F4320322436","display_name":"Novo Nordisk","ror":"https://ror.org/0435rc536"},{"id":"https://openalex.org/F4320325957","display_name":"Novo Nordisk Fonden","ror":"https://ror.org/04txyc737"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"How":[0],"do":[1],"product":[2],"teams":[3],"evaluate":[4],"LLM-powered":[5],"products?":[6],"As":[7],"organizations":[8],"integrate":[9],"large":[10],"language":[11],"models":[12],"(LLMs)":[13],"into":[14,80],"digital":[15],"products,":[16],"their":[17],"unpredictable":[18],"nature":[19],"makes":[20],"traditional":[21],"evaluation":[22,46,74,145],"approaches":[23],"inadequate,":[24],"yet":[25],"little":[26],"is":[27],"known":[28],"about":[29],"how":[30],"practitioners":[31,39,72,136],"navigate":[32],"this":[33,94,129],"challenge.":[34],"Through":[35],"interviews":[36],"with":[37],"nineteen":[38],"across":[40],"diverse":[41],"sectors,":[42],"we":[43,60,65,89],"identify":[44],"ten":[45],"practices":[47,103,115,140],"spanning":[48],"informal":[49],"\u2018vibe":[50],"checks\u2019":[51],"to":[52,92,119,134],"organizational":[53],"meta-work.":[54],"Beyond":[55],"confirming":[56],"four":[57],"documented":[58],"challenges,":[59],"introduce":[61],"a":[62,131],"novel":[63],"fifth":[64],"call":[66],"the":[67],"results-actionability":[68],"gap,":[69,95],"in":[70,137],"which":[71],"gather":[73],"data":[75],"but":[76],"cannot":[77],"translate":[78],"findings":[79],"concrete":[81],"improvements.":[82],"Drawing":[83],"on":[84],"patterns":[85],"from":[86,100],"successful":[87],"teams,":[88],"contribute":[90],"strategies":[91],"bridge":[93],"supporting":[96],"practitioners\u2019":[97],"formalization":[98],"journey":[99],"ad-hoc":[101],"interpretive":[102,114],"(e.g.,":[104],"vibe":[105],"checks)":[106],"toward":[107],"systematic":[108],"evaluation.":[109],"Our":[110],"analysis":[111],"suggests":[112],"these":[113],"are":[116],"necessary":[117],"adaptations":[118],"LLM":[120],"characteristics":[121],"rather":[122,141],"than":[123,142],"methodological":[124],"failures.":[125],"For":[126],"HCI":[127],"researchers,":[128],"presents":[130],"research":[132],"opportunity":[133],"support":[135],"systematizing":[138],"emerging":[139],"developing":[143],"new":[144],"frameworks.":[146]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2026-03-07T00:00:00"}