{"id":"https://openalex.org/W4396812188","doi":"https://doi.org/10.1145/3626772.3657846","title":"On the Evaluation of Machine-Generated Reports","display_name":"On the Evaluation of Machine-Generated Reports","publication_year":2024,"publication_date":"2024-07-10","ids":{"openalex":"https://openalex.org/W4396812188","doi":"https://doi.org/10.1145/3626772.3657846"},"language":"en","primary_location":{"id":"doi:10.1145/3626772.3657846","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657846","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3626772.3657846","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036270568","display_name":"James Mayfield","orcid":"https://orcid.org/0000-0003-3866-3013"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"James Mayfield","raw_affiliation_strings":["Johns Hopkins University, Baltimore, MD, USA","Allen Institute for AI Seattle, WA, USA","University of Maryland College Park, MD, USA","University of Glasgow Glasgow, United Kingdom","NIST Gaithersburg, MD, USA"],"raw_orcid":"https://orcid.org/0000-0003-3866-3013","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062016266","display_name":"Eugene Yang","orcid":"https://orcid.org/0000-0002-0051-1535"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Eugene Yang","raw_affiliation_strings":["Johns Hopkins University, Baltimore, MD, USA","NIST Gaithersburg, MD, USA","University of Maryland College Park, MD, USA","University of Glasgow Glasgow, United Kingdom","Allen Institute for AI Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-0051-1535","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037074105","display_name":"Dawn Lawrie","orcid":"https://orcid.org/0000-0001-7347-7086"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Dawn Lawrie","raw_affiliation_strings":["Johns Hopkins University, Baltimore, MD, USA","NIST Gaithersburg, MD, USA","Allen Institute for AI Seattle, WA, USA","University of Maryland College Park, MD, USA","University of Glasgow Glasgow, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0001-7347-7086","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014199889","display_name":"Sean MacAvaney","orcid":"https://orcid.org/0000-0002-8914-2659"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Sean MacAvaney","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom","Allen Institute for AI Seattle, WA, USA","Johns Hopkins University Baltimore, MD, USA","NIST Gaithersburg, MD, USA","University of Maryland College Park, MD, USA"],"raw_orcid":"https://orcid.org/0000-0002-8914-2659","affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"Johns Hopkins University Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006760609","display_name":"Paul McNamee","orcid":"https://orcid.org/0000-0002-0548-5751"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Paul McNamee","raw_affiliation_strings":["Johns Hopkins University, Baltimore, MD, USA","NIST Gaithersburg, MD, USA","University of Glasgow Glasgow, United Kingdom","Allen Institute for AI Seattle, WA, USA","University of Maryland College Park, MD, USA"],"raw_orcid":"https://orcid.org/0000-0002-0548-5751","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074322406","display_name":"Douglas W. Oard","orcid":"https://orcid.org/0000-0002-1696-0407"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Douglas W. Oard","raw_affiliation_strings":["University of Maryland, College Park, MD, USA","University of Glasgow Glasgow, United Kingdom","Johns Hopkins University Baltimore, MD, USA","NIST Gaithersburg, MD, USA","Allen Institute for AI Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-1696-0407","affiliations":[{"raw_affiliation_string":"University of Maryland, College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"Johns Hopkins University Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060844217","display_name":"Luca Soldaini","orcid":"https://orcid.org/0000-0001-6998-9863"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Luca Soldaini","raw_affiliation_strings":["Allen Institute for AI, Seattle, WA, USA","University of Glasgow Glasgow, United Kingdom","Johns Hopkins University Baltimore, MD, USA","University of Maryland College Park, MD, USA","NIST Gaithersburg, MD, USA"],"raw_orcid":"https://orcid.org/0000-0001-6998-9863","affiliations":[{"raw_affiliation_string":"Allen Institute for AI, Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"Johns Hopkins University Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053747568","display_name":"Ian Soboroff","orcid":"https://orcid.org/0000-0003-2363-3014"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Ian Soboroff","raw_affiliation_strings":["NIST, Gaithersburg, MD, USA","Johns Hopkins University Baltimore, MD, USA","Allen Institute for AI Seattle, WA, USA","University of Glasgow Glasgow, United Kingdom","University of Maryland College Park, MD, USA"],"raw_orcid":"https://orcid.org/0000-0003-2363-3014","affiliations":[{"raw_affiliation_string":"NIST, Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"Johns Hopkins University Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056096860","display_name":"Orion Weller","orcid":"https://orcid.org/0000-0003-4148-5430"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Orion Weller","raw_affiliation_strings":["Johns Hopkins University, Baltimore, MD, USA","NIST Gaithersburg, MD, USA","University of Maryland College Park, MD, USA","University of Glasgow Glasgow, United Kingdom","Allen Institute for AI Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0003-4148-5430","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060057841","display_name":"Efsun Sarioglu Kayi","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Efsun Kayi","raw_affiliation_strings":["Johns Hopkins University, Baltimore, MD, USA","University of Glasgow Glasgow, United Kingdom","NIST Gaithersburg, MD, USA","University of Maryland College Park, MD, USA","Allen Institute for AI Seattle, WA, USA"],"raw_orcid":"https://orcid.org/0000-0002-2357-9807","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102102804","display_name":"Kate Sanders","orcid":"https://orcid.org/0000-0003-3125-2975"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Kate Sanders","raw_affiliation_strings":["Johns Hopkins University, Baltimore, MD, USA","University of Maryland College Park, MD, USA","Allen Institute for AI Seattle, WA, USA","University of Glasgow Glasgow, United Kingdom","NIST Gaithersburg, MD, USA"],"raw_orcid":"https://orcid.org/0000-0003-3125-2975","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015696460","display_name":"Marc A. Mason","orcid":"https://orcid.org/0009-0001-1784-6526"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Marc Mason","raw_affiliation_strings":["Johns Hopkins University, Baltimore, MD, USA","NIST Gaithersburg, MD, USA","University of Glasgow Glasgow, United Kingdom","Allen Institute for AI Seattle, WA, USA","University of Maryland College Park, MD, USA"],"raw_orcid":"https://orcid.org/0009-0001-1784-6526","affiliations":[{"raw_affiliation_string":"Johns Hopkins University, Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"University of Maryland College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5097335480","display_name":"Noah Hibbler","orcid":"https://orcid.org/0009-0001-7967-4109"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]},{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]},{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Noah Hibbler","raw_affiliation_strings":["University of Maryland, College Park, MD, USA","Johns Hopkins University Baltimore, MD, USA","Allen Institute for AI Seattle, WA, USA","NIST Gaithersburg, MD, USA","University of Glasgow Glasgow, United Kingdom"],"raw_orcid":"https://orcid.org/0009-0001-7967-4109","affiliations":[{"raw_affiliation_string":"University of Maryland, College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]},{"raw_affiliation_string":"Johns Hopkins University Baltimore, MD, USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Allen Institute for AI Seattle, WA, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"NIST Gaithersburg, MD, USA","institution_ids":[]},{"raw_affiliation_string":"University of Glasgow Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":25,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1904","last_page":"1915"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6818300485610962}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6818300485610962}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3626772.3657846","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657846","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2405.00982","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.00982","pdf_url":"https://arxiv.org/pdf/2405.00982","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:eprints.gla.ac.uk:323367","is_oa":true,"landing_page_url":null,"pdf_url":"https://eprints.gla.ac.uk/323367/1/323367.pdf","source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Proceedings"}],"best_oa_location":{"id":"doi:10.1145/3626772.3657846","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3626772.3657846","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 47th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1507150160","https://openalex.org/W1989420837","https://openalex.org/W1990190154","https://openalex.org/W2005589065","https://openalex.org/W2051167396","https://openalex.org/W2075893676","https://openalex.org/W2081174553","https://openalex.org/W2092840709","https://openalex.org/W2150824314","https://openalex.org/W2154407881","https://openalex.org/W2251023345","https://openalex.org/W2467173223","https://openalex.org/W2598654328","https://openalex.org/W2751013112","https://openalex.org/W2783389297","https://openalex.org/W2798533112","https://openalex.org/W2798542795","https://openalex.org/W2952317054","https://openalex.org/W2953642885","https://openalex.org/W2980420460","https://openalex.org/W3027879771","https://openalex.org/W3035259499","https://openalex.org/W3101098106","https://openalex.org/W3103991399","https://openalex.org/W3151369355","https://openalex.org/W3155807546","https://openalex.org/W4214900817","https://openalex.org/W4362471045","https://openalex.org/W4386065803","https://openalex.org/W4386365475","https://openalex.org/W4389518925","https://openalex.org/W4391156274","https://openalex.org/W6600655081","https://openalex.org/W6631918127"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4,15],"enabled":[5],"new":[6,159],"ways":[7],"to":[8,21,33,47,78,148,196,215],"satisfy":[9,48],"information":[10,54,112],"needs.":[11],"Although":[12],"great":[13],"strides":[14],"been":[16],"made":[17,211],"in":[18,161,175,212],"applying":[19],"them":[20],"settings":[22],"like":[23],"document":[24],"ranking":[25],"and":[26,37,68,70,119,132,150,181,192],"short-form":[27],"text":[28],"generation,":[29,85],"they":[30],"still":[31],"struggle":[32],"compose":[34],"complete,":[35,130],"accurate,":[36,131],"verifiable":[38],"long-form":[39],"reports.":[40],"Reports":[41],"with":[42,98,106],"these":[43,155,163],"qualities":[44],"are":[45,137],"necessary":[46,116],"the":[49,115,122,125,183,213],"complex,":[50],"nuanced,":[51],"or":[52],"multi-faceted":[53],"needs":[55],"of":[56,74,110,121,187,199,206],"users.":[57],"In":[58,96],"this":[59],"perspective":[60],"paper,":[61],"we":[62,165],"draw":[63],"together":[64],"opinions":[65],"from":[66,71],"industry":[67],"academia,":[69],"a":[72,107],"variety":[73],"related":[75],"research":[76],"areas,":[77],"present":[79,166],"our":[80],"vision":[81],"for":[82],"automatic":[83,102],"report":[84,103,214],"and---critically---a":[86],"flexible":[87],"framework":[88,169,184],"by":[89],"which":[90,136],"such":[91],"reports":[92,127],"can":[93],"be":[94,129,197],"evaluated.":[95],"contrast":[97],"other":[99],"summarization":[100],"tasks,":[101],"generation":[104],"starts":[105],"detailed":[108],"description":[109],"an":[111,167],"need,":[113],"stating":[114],"background,":[117],"requirements,":[118],"scope":[120],"report.":[123,203],"Further,":[124],"generated":[126,202],"should":[128],"verifiable.":[133],"These":[134],"qualities,":[135],"desirable---if":[138],"not":[139],"required---in":[140],"many":[141],"analytic":[142],"report-writing":[143],"settings,":[144],"require":[145],"rethinking":[146],"how":[147],"build":[149],"evaluate":[151],"systems":[152],"that":[153,170,194,208],"exhibit":[154],"qualities.":[156],"To":[157,178],"foster":[158],"efforts":[160],"building":[162],"systems,":[164],"evaluation":[168,205],"draws":[171],"on":[172],"ideas":[173],"found":[174],"various":[176],"evaluations.":[177],"test":[179],"completeness":[180],"accuracy,":[182],"uses":[185],"nuggets":[186],"information,":[188],"expressed":[189],"as":[190],"questions":[191],"answers,":[193],"need":[195],"part":[198],"any":[200],"high-quality":[201],"Additionally,":[204],"citations":[207],"map":[209],"claims":[210],"their":[216],"source":[217],"documents":[218],"ensures":[219],"verifiability.":[220]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2025-10-10T00:00:00"}