{"id":"https://openalex.org/W4392617724","doi":"https://doi.org/10.1145/3640543.3645142","title":"iScore: Visual Analytics for Interpreting How Language Models Automatically Score Summaries","display_name":"iScore: Visual Analytics for Interpreting How Language Models Automatically Score Summaries","publication_year":2024,"publication_date":"2024-03-18","ids":{"openalex":"https://openalex.org/W4392617724","doi":"https://doi.org/10.1145/3640543.3645142"},"language":"en","primary_location":{"id":"doi:10.1145/3640543.3645142","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645142","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645142","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645142","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053775923","display_name":"Adam Coscia","orcid":"https://orcid.org/0000-0002-0429-9295"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Adam Coscia","raw_affiliation_strings":["Georgia Institute of Technology, United States"],"raw_orcid":"https://orcid.org/0000-0002-0429-9295","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, United States","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090861447","display_name":"Langdon Holmes","orcid":"https://orcid.org/0000-0003-4338-4609"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Langdon Holmes","raw_affiliation_strings":["Vanderbilt University, United States"],"raw_orcid":"https://orcid.org/0000-0003-4338-4609","affiliations":[{"raw_affiliation_string":"Vanderbilt University, United States","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047788899","display_name":"Wesley Morris","orcid":"https://orcid.org/0000-0001-6316-6479"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wesley Morris","raw_affiliation_strings":["Vanderbilt University, United States"],"raw_orcid":"https://orcid.org/0000-0001-6316-6479","affiliations":[{"raw_affiliation_string":"Vanderbilt University, United States","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070821758","display_name":"Joon Suh Choi","orcid":"https://orcid.org/0000-0002-7732-0366"},"institutions":[{"id":"https://openalex.org/I181565077","display_name":"Georgia State University","ror":"https://ror.org/03qt6ba18","country_code":"US","type":"education","lineage":["https://openalex.org/I181565077"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joon Suh Choi","raw_affiliation_strings":["Applied Linguistics, Georgia State University, United States"],"raw_orcid":"https://orcid.org/0000-0002-7732-0366","affiliations":[{"raw_affiliation_string":"Applied Linguistics, Georgia State University, United States","institution_ids":["https://openalex.org/I181565077"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028338673","display_name":"Scott A. Crossley","orcid":"https://orcid.org/0000-0002-5148-0273"},"institutions":[{"id":"https://openalex.org/I200719446","display_name":"Vanderbilt University","ror":"https://ror.org/02vm5rt34","country_code":"US","type":"education","lineage":["https://openalex.org/I200719446"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Scott Crossley","raw_affiliation_strings":["Vanderbilt University, United States"],"raw_orcid":"https://orcid.org/0000-0002-5148-0273","affiliations":[{"raw_affiliation_string":"Vanderbilt University, United States","institution_ids":["https://openalex.org/I200719446"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064306421","display_name":"Alex Endert","orcid":"https://orcid.org/0000-0002-6914-610X"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alex Endert","raw_affiliation_strings":["Georgia Institute of Technology, United States"],"raw_orcid":"https://orcid.org/0000-0002-6914-610X","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, United States","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5053775923"],"corresponding_institution_ids":["https://openalex.org/I130701444"],"apc_list":null,"apc_paid":null,"fwci":6.714,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.96559403,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"787","last_page":"802"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.758440375328064},{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.6203316450119019},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.5729500651359558},{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.5177704095840454},{"id":"https://openalex.org/keywords/learning-analytics","display_name":"Learning analytics","score":0.5161513090133667},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4940205216407776},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4572559893131256},{"id":"https://openalex.org/keywords/visual-analytics","display_name":"Visual analytics","score":0.45486563444137573},{"id":"https://openalex.org/keywords/upload","display_name":"Upload","score":0.45036599040031433},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.4473077654838562},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.43533506989479065},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.410250186920166},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.3942795395851135},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.23433426022529602}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.758440375328064},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.6203316450119019},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.5729500651359558},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.5177704095840454},{"id":"https://openalex.org/C2777648619","wikidata":"https://www.wikidata.org/wiki/Q2845208","display_name":"Learning analytics","level":2,"score":0.5161513090133667},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4940205216407776},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4572559893131256},{"id":"https://openalex.org/C59732488","wikidata":"https://www.wikidata.org/wiki/Q2528440","display_name":"Visual analytics","level":3,"score":0.45486563444137573},{"id":"https://openalex.org/C71901391","wikidata":"https://www.wikidata.org/wiki/Q7126699","display_name":"Upload","level":2,"score":0.45036599040031433},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.4473077654838562},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.43533506989479065},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.410250186920166},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.3942795395851135},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.23433426022529602},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3640543.3645142","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645142","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645142","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2403.04760","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.04760","pdf_url":"https://arxiv.org/pdf/2403.04760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3640543.3645142","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3640543.3645142","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3640543.3645142","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th International Conference on Intelligent User Interfaces","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.8199999928474426,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1146492981","display_name":null,"funder_award_id":"2247790","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3723102056","display_name":null,"funder_award_id":"2247790, 2112532","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"},{"id":"https://openalex.org/G4486241280","display_name":null,"funder_award_id":"2112532","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392617724.pdf","grobid_xml":"https://content.openalex.org/works/W4392617724.grobid-xml"},"referenced_works_count":41,"referenced_works":["https://openalex.org/W1951008065","https://openalex.org/W1970569592","https://openalex.org/W2053075547","https://openalex.org/W2135415614","https://openalex.org/W2144988340","https://openalex.org/W2280974225","https://openalex.org/W2282821441","https://openalex.org/W2301632029","https://openalex.org/W2587299461","https://openalex.org/W2602387228","https://openalex.org/W2618851150","https://openalex.org/W2733082571","https://openalex.org/W2794802448","https://openalex.org/W2883454655","https://openalex.org/W2898193199","https://openalex.org/W2947964661","https://openalex.org/W2953001123","https://openalex.org/W2962862931","https://openalex.org/W2963123635","https://openalex.org/W2963214037","https://openalex.org/W2966491090","https://openalex.org/W2981852735","https://openalex.org/W3011616684","https://openalex.org/W3013371068","https://openalex.org/W3021941048","https://openalex.org/W3087302936","https://openalex.org/W3093452197","https://openalex.org/W3118485687","https://openalex.org/W3133702157","https://openalex.org/W3159958158","https://openalex.org/W3175479236","https://openalex.org/W4207045419","https://openalex.org/W4224229986","https://openalex.org/W4281790347","https://openalex.org/W4288059323","https://openalex.org/W4288089799","https://openalex.org/W4293464023","https://openalex.org/W4294663947","https://openalex.org/W4308262294","https://openalex.org/W4318464200","https://openalex.org/W4382567099"],"related_works":["https://openalex.org/W1987827786","https://openalex.org/W2799586942","https://openalex.org/W2504091800","https://openalex.org/W2331775400","https://openalex.org/W2816728186","https://openalex.org/W2804624249","https://openalex.org/W2560130217","https://openalex.org/W605203981","https://openalex.org/W2911048623","https://openalex.org/W4292348877"],"abstract_inverted_index":{"The":[0],"recent":[1],"explosion":[2],"in":[3,35,129,133,208],"popularity":[4],"of":[5,46,145,161],"large":[6,85],"language":[7,128],"models":[8],"(LLMs)":[9],"has":[10],"inspired":[11],"learning":[12,37,64,109,156,175,194],"engineers":[13,65,110,157,195],"to":[14,111,124,177,202],"incorporate":[15],"them":[16,34,201],"into":[17],"adaptive":[18],"educational":[19],"tools":[20],"that":[21,196],"automatically":[22],"score":[23,89,181],"summary":[24,69],"writing.":[25],"Understanding":[26],"and":[27,43,50,67,77,91,114,138,205],"evaluating":[28],"LLMs":[29,210],"is":[30],"vital":[31],"before":[32],"deploying":[33,68],"critical":[36],"environments,":[38],"yet":[39],"their":[40,81,98,179,209],"unprecedented":[41],"size":[42],"expanding":[44],"number":[45],"parameters":[47],"inhibits":[48],"transparency":[49],"impedes":[51],"trust":[52,207],"when":[53],"they":[54],"underperform.":[55],"Through":[56],"a":[57,162,166,174],"collaborative":[58],"user-centered":[59],"design":[60,75],"process":[61],"with":[62,154,171,192],"several":[63],"building":[66],"scoring":[70],"LLMs,":[71],"we":[72,100,151,188],"characterized":[73],"fundamental":[74],"challenges":[76],"goals":[78],"around":[79],"interpreting":[80],"models,":[82],"including":[83],"aggregating":[84],"text":[86],"inputs,":[87],"tracking":[88],"provenance,":[90],"scaling":[92],"LLM":[93,136],"interpretability":[94],"methods.":[95],"To":[96,147],"address":[97],"concerns,":[99],"developed":[101],"iScore,":[102],"an":[103],"interactive":[104],"visual":[105],"analytics":[106],"tool":[107],"for":[108],"upload,":[112],"score,":[113],"compare":[115],"multiple":[116,143],"summaries":[117],"simultaneously.":[118],"Tightly":[119],"integrated":[120],"views":[121],"allow":[122],"users":[123],"iteratively":[125],"revise":[126],"the":[127,134,159,193],"summaries,":[130],"track":[131],"changes":[132],"resulting":[135],"scores,":[137],"visualize":[139],"model":[140],"weights":[141],"at":[142],"levels":[144],"abstraction.":[146],"validate":[148],"our":[149],"approach,":[150],"deployed":[152],"iScore":[153,172,199],"three":[155,184],"over":[158],"course":[160],"month.":[163],"We":[164],"present":[165],"case":[167],"study":[168],"where":[169],"interacting":[170],"led":[173],"engineer":[176],"improve":[178],"LLM\u2019s":[180],"accuracy":[182],"by":[183],"percentage":[185],"points.":[186],"Finally,":[187],"conducted":[189],"qualitative":[190],"interviews":[191],"revealed":[197],"how":[198],"enabled":[200],"understand,":[203],"evaluate,":[204],"build":[206],"during":[211],"deployment.":[212]},"counts_by_year":[{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}