{"id":"https://openalex.org/W4408550774","doi":"https://doi.org/10.1145/3708557.3716146","title":"Evaluation Workflows for Large Language Models (LLMs) that Integrate Domain Expertise for Complex Knowledge Tasks","display_name":"Evaluation Workflows for Large Language Models (LLMs) that Integrate Domain Expertise for Complex Knowledge Tasks","publication_year":2025,"publication_date":"2025-03-18","ids":{"openalex":"https://openalex.org/W4408550774","doi":"https://doi.org/10.1145/3708557.3716146"},"language":"en","primary_location":{"id":"doi:10.1145/3708557.3716146","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3708557.3716146","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3708557.3716146?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the 30th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3708557.3716146?download=true","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093206754","display_name":"Annalisa Szymanski","orcid":null},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Annalisa Szymanski","raw_affiliation_strings":["Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA"],"raw_orcid":"https://orcid.org/0009-0009-5472-282X","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering, University of Notre Dame, Notre Dame, Indiana, USA","institution_ids":["https://openalex.org/I107639228"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5093206754"],"corresponding_institution_ids":["https://openalex.org/I107639228"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"215","last_page":"217"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9828000068664551,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.75364089012146},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.7499165534973145},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5790424346923828},{"id":"https://openalex.org/keywords/domain-knowledge","display_name":"Domain knowledge","score":0.4533412754535675},{"id":"https://openalex.org/keywords/domain-specific-language","display_name":"Domain-specific language","score":0.42197859287261963},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.4146064817905426},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.37059569358825684},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3465602993965149},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.28502726554870605},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.08801144361495972}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.75364089012146},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.7499165534973145},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5790424346923828},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.4533412754535675},{"id":"https://openalex.org/C135257023","wikidata":"https://www.wikidata.org/wiki/Q691358","display_name":"Domain-specific language","level":2,"score":0.42197859287261963},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4146064817905426},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37059569358825684},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3465602993965149},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.28502726554870605},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.08801144361495972},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3708557.3716146","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3708557.3716146","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3708557.3716146?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the 30th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3708557.3716146","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3708557.3716146","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3708557.3716146?download=true","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the 30th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5199999809265137}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4408550774.pdf","grobid_xml":"https://content.openalex.org/works/W4408550774.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W3034238904","https://openalex.org/W4367310920","https://openalex.org/W4378908626","https://openalex.org/W4385605076","https://openalex.org/W4388196756","https://openalex.org/W4396832834","https://openalex.org/W4396833346","https://openalex.org/W4396833396","https://openalex.org/W4396833505","https://openalex.org/W4399521940","https://openalex.org/W4402683144","https://openalex.org/W4403334020","https://openalex.org/W4404783381"],"related_works":["https://openalex.org/W2057033141","https://openalex.org/W4252103056","https://openalex.org/W2917823318","https://openalex.org/W2917572814","https://openalex.org/W4242858628","https://openalex.org/W2979406678","https://openalex.org/W2915857720","https://openalex.org/W2916715723","https://openalex.org/W1583422155","https://openalex.org/W3006129753"],"abstract_inverted_index":{"The":[0],"growing":[1],"use":[2],"of":[3,28,39,65,80,105,137,145,151,158],"Large":[4],"Language":[5],"Models":[6],"(LLMs)":[7],"in":[8,58,119,167,179],"specialized":[9],"fields":[10],"such":[11,112],"as":[12,113],"healthcare,":[13],"nutrition,":[14],"and":[15,25,42,52,88,116,123,129,140,162,176],"education":[16],"has":[17],"raised":[18],"critical":[19],"concerns":[20],"regarding":[21],"the":[22,37,40,43,56,63,103,120,134,142,156],"accuracy,":[23],"reliability,":[24],"contextual":[26],"appropriateness":[27],"LLM":[29,66,98,159],"outputs.However,":[30],"evaluating":[31],"LLMs":[32,117,178],"is":[33,49,171],"challenging":[34],"due":[35],"to":[36,72,85,101,109,125,173],"complexity":[38],"information":[41],"need":[44],"for":[45,68],"human":[46],"input,":[47],"which":[48],"often":[50],"costly":[51],"resource-intensive.My":[53],"dissertation":[54],"addresses":[55],"challenges":[57],"integrating":[59],"domain":[60,92,106,138],"expertise":[61,93,170],"into":[62,97],"evaluation":[64,76,99,121,127,147,160],"outputs":[67],"complex":[69],"knowledge":[70],"tasks":[71],"build":[73],"more":[74],"efficient":[75],"workflows.The":[77],"main":[78],"objectives":[79],"this":[81,152],"research":[82,153],"are:":[83],"1)":[84],"investigate":[86],"when":[87],"at":[89],"what":[90],"stage":[91],"should":[94],"be":[95],"integrated":[96],"2)":[100],"explore":[102],"role":[104],"experts":[107,139],"compared":[108],"other":[110,146],"evaluators":[111],"lay":[114],"users":[115],"themselves":[118],"process,":[122],"3)":[124],"design":[126,157],"frameworks":[128],"tools":[130,161],"that":[131,164],"guide":[132],"both":[133],"optimal":[135],"integration":[136],"leverage":[141],"complementary":[143],"strengths":[144],"groups.The":[148],"expected":[149],"impact":[150],"includes":[154],"advancing":[155],"workflows":[163],"assist":[165],"developers":[166],"identifying":[168],"where":[169],"needed":[172],"effectively":[174],"develop":[175],"deploy":[177],"real-world":[180],"applications.":[181],"CCS":[182],"Concepts":[183]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}