{"id":"https://openalex.org/W4400530685","doi":"https://doi.org/10.1145/3626772.3657871","title":"A Workbench for Autograding Retrieve/Generate Systems","display_name":"A Workbench for Autograding Retrieve/Generate Systems","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4400530685","doi":"https://doi.org/10.1145/3626772.3657871"},"language":"en","primary_location":{"id":"doi:10.1145/3626772.3657871","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626772.3657871","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027260515","display_name":"Laura Dietz","orcid":"https://orcid.org/0000-0003-1624-3907"},"institutions":[{"id":"https://openalex.org/I161057412","display_name":"University of New Hampshire","ror":"https://ror.org/01rmh9n78","country_code":"US","type":"education","lineage":["https://openalex.org/I161057412"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Laura Dietz","raw_affiliation_strings":["University of New Hampshire, Durham, NH, USA"],"raw_orcid":"https://orcid.org/0000-0003-1624-3907","affiliations":[{"raw_affiliation_string":"University of New Hampshire, Durham, NH, USA","institution_ids":["https://openalex.org/I161057412"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5027260515"],"corresponding_institution_ids":["https://openalex.org/I161057412"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1963","last_page":"1972"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9832000136375427,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9832000136375427,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9745000004768372,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12288","display_name":"Optimization and Search Problems","score":0.9631999731063843,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workbench","display_name":"Workbench","score":0.8665022253990173},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7640647888183594},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24600249528884888},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.12250915169715881}],"concepts":[{"id":"https://openalex.org/C2779145975","wikidata":"https://www.wikidata.org/wiki/Q347690","display_name":"Workbench","level":3,"score":0.8665022253990173},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7640647888183594},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24600249528884888},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.12250915169715881}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3626772.3657871","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3626772.3657871","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1493399377","https://openalex.org/W2017633929","https://openalex.org/W2021420765","https://openalex.org/W2078861931","https://openalex.org/W2097193844","https://openalex.org/W2100709351","https://openalex.org/W2113474780","https://openalex.org/W2132314908","https://openalex.org/W2166455213","https://openalex.org/W2461068311","https://openalex.org/W2945760033","https://openalex.org/W3034188538","https://openalex.org/W3035643691","https://openalex.org/W3138773240","https://openalex.org/W4221143046","https://openalex.org/W4385688511","https://openalex.org/W4389950508"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2388509490","https://openalex.org/W2109648960","https://openalex.org/W2166015079","https://openalex.org/W2050384624","https://openalex.org/W2776888796","https://openalex.org/W2388480422","https://openalex.org/W2624268110","https://openalex.org/W2951120646"],"abstract_inverted_index":{"This":[0,102],"resource":[1],"paper":[2],"addresses":[3],"the":[4,13,33,53,67,73,86,90,100,107],"challenge":[5],"of":[6,15,35,55,77,96,109,119],"evaluating":[7],"Information":[8],"Retrieval":[9],"(IR)":[10],"systems":[11],"in":[12,85],"era":[14],"autoregressive":[16],"Large":[17],"Language":[18],"Models":[19],"(LLMs).":[20],"Traditional":[21],"methods":[22],"relying":[23],"on":[24,127],"passage-level":[25],"judgments":[26],"are":[27],"no":[28],"longer":[29],"effective":[30],"due":[31],"to":[32,45,51,92,105],"diversity":[34],"responses":[36],"generated":[37],"by":[38],"LLM-based":[39],"systems.":[40],"We":[41],"provide":[42],"a":[43,56,94],"workbench":[44,103],"explore":[46],"several":[47],"alternative":[48],"evaluation":[49,129],"approaches":[50],"judge":[52],"relevance":[54],"system's":[57],"response":[58,68],"that":[59],"incorporate":[60],"LLMs:":[61],"1.":[62],"Asking":[63,72,89],"an":[64],"LLM":[65,74,91],"whether":[66],"is":[69,83],"relevant;":[70],"2.":[71],"which":[75],"set":[76,95],"nuggets":[78,120],"(i.e.,":[79],"relevant":[80],"key":[81],"facts)":[82],"covered":[84],"response;":[87],"3.":[88],"answer":[93],"exam":[97,122],"questions":[98],"with":[99],"response.":[101],"aims":[104],"facilitate":[106],"development":[108],"new,":[110],"reusable":[111],"test":[112],"collections.":[113],"Researchers":[114],"can":[115],"manually":[116],"refine":[117],"sets":[118],"and":[121,130],"questions,":[123],"observing":[124],"their":[125],"impact":[126],"system":[128],"leaderboard":[131],"rankings.":[132],"Resource":[133],"available":[134],"at":[135],"https://github.com/TREMA-UNH/rubric-grading-workbench":[136]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}