{"id":"https://openalex.org/W7160532358","doi":"https://doi.org/10.48550/arxiv.2605.04458","title":"DoGMaTiQ: Automated Generation of Question-and-Answer Nuggets for Report Evaluation","display_name":"DoGMaTiQ: Automated Generation of Question-and-Answer Nuggets for Report Evaluation","publication_year":2026,"publication_date":"2026-05-06","ids":{"openalex":"https://openalex.org/W7160532358","doi":"https://doi.org/10.48550/arxiv.2605.04458"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.04458","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.04458","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.04458","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135613277","display_name":"Bryan Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Bryan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5097607379","display_name":"William Walden","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Walden, William","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135549615","display_name":"Yu Hou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hou, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107941641","display_name":"G. Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Gabrielle Kaili-May","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135542923","display_name":"Dawn Lawrie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lawrie, Dawn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135580158","display_name":"Jame Mayfield","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mayfield, James","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135614224","display_name":"Eugene Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Eugene","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135596738","display_name":"Chris Callison-Burch","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Callison-Burch, Chris","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135563645","display_name":"Laura Dietz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dietz, Laura","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6122999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6122999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.11240000277757645,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.05640000104904175,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.7059999704360962},{"id":"https://openalex.org/keywords/paraphrase","display_name":"Paraphrase","score":0.5432000160217285},{"id":"https://openalex.org/keywords/generator","display_name":"Generator (circuit theory)","score":0.5249000191688538},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4918999969959259},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4593999981880188},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.4341999888420105},{"id":"https://openalex.org/keywords/core","display_name":"Core (optical fiber)","score":0.3815000057220459},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.3752000033855438}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7312999963760376},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.7059999704360962},{"id":"https://openalex.org/C2780922921","wikidata":"https://www.wikidata.org/wiki/Q255189","display_name":"Paraphrase","level":2,"score":0.5432000160217285},{"id":"https://openalex.org/C2780992000","wikidata":"https://www.wikidata.org/wiki/Q17016113","display_name":"Generator (circuit theory)","level":3,"score":0.5249000191688538},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4918999969959259},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4593999981880188},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.4341999888420105},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4300000071525574},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4163999855518341},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.3815000057220459},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3752000033855438},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3515999913215637},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.34880000352859497},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.30959999561309814},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.30480000376701355},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.30300000309944153},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2921999990940094},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.2904999852180481},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.2842999994754791},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.2678999900817871},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2538999915122986}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.04458","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.04458","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.04458","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.04458","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5079944133758545,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Evaluation":[0],"of":[1,15,28,38,97,181,211],"long-form,":[2],"citation-backed":[3],"reports":[4],"has":[5,57],"lately":[6],"received":[7],"significant":[8],"attention":[9],"due":[10],"to":[11,21,33,93,113,176,233],"the":[12,26,43,67,71,74,91,225],"wide-scale":[13],"adoption":[14],"retrieval-augmented":[16],"generation":[17],"(RAG)":[18],"systems.":[19,235],"Core":[20],"many":[22],"evaluation":[23,89,173,180,242],"frameworks":[24],"is":[25,90,119,126,221],"use":[27],"atomic":[29],"facts,":[30],"or":[31],"nuggets,":[32],"assess":[34],"a":[35,103,107,136,170,216],"report's":[36],"coverage":[37],"query-relevant":[39],"information":[40,68,115,125],"attested":[41],"in":[42,102,121,128,144,240],"underlying":[44],"collection.":[45],"While":[46],"nuggets":[47,98,166],"have":[48],"traditionally":[49],"been":[50],"represented":[51],"as":[52],"short":[53],"statements,":[54],"recent":[55,171],"work":[56],"used":[58],"question-answer":[59],"(QA)":[60],"representations,":[61],"enabling":[62],"fine-grained":[63],"evaluations":[64],"that":[65,78,110,215,224],"decouple":[66],"need":[69,92],"(i.e.":[70,81],"question)":[72],"from":[73],"potentially":[75],"diverse":[76],"content":[77],"satisfies":[79],"it":[80],"its":[82],"answers).":[83],"A":[84],"persistent":[85],"challenge":[86,118],"for":[87,99,138],"nugget-based":[88,172],"manually":[94],"curate":[95],"sets":[96,143],"each":[100],"topic":[101],"test":[104],"collection":[105],"--":[106,169,175],"laborious":[108],"process":[109],"scales":[111],"poorly":[112],"novel":[114],"needs.":[116],"This":[117],"acute":[120],"cross-lingual":[122,190],"settings,":[123],"where":[124],"found":[127],"multilingual":[129],"source":[130],"documents.":[131],"Accordingly,":[132],"we":[133],"introduce":[134],"DoGMaTiQ,":[135],"pipeline":[137,213],"generating":[139],"high-quality":[140],"QA-based":[141],"nugget":[142,149,156,219],"three":[145],"stages:":[146],"(1)":[147],"document-grounded":[148],"generation,":[150],"(2)":[151],"paraphrase":[152],"clustering,":[153],"and":[154,195,204,223,248],"(3)":[155],"subselection":[157],"based":[158],"on":[159,188],"principled":[160],"quality":[161],"criteria.":[162],"We":[163,184,236],"integrate":[164],"DoGMaTiQ":[165,230],"with":[167,201],"AutoArgue":[168],"framework":[174],"enable":[177],"fully":[178,205],"automatic":[179],"generated":[182],"reports.":[183],"conduct":[185],"extensive":[186],"experiments":[187],"two":[189],"TREC":[191],"shared":[192],"tasks,":[193],"NeuCLIR":[194],"RAGTIME,":[196],"showing":[197],"strong":[198,217],"rank":[199],"correlations":[200],"both":[202],"human-in-the-loop":[203],"manual":[206],"judgments.":[207],"Finally,":[208],"detailed":[209],"analysis":[210],"our":[212,246],"reveals":[214],"LLM":[218],"generator":[220],"key,":[222],"system":[226],"rankings":[227],"induced":[228],"by":[229,243],"are":[231],"robust":[232],"outlier":[234],"facilitate":[237],"future":[238],"research":[239],"report":[241],"publicly":[244],"releasing":[245],"code":[247],"artifacts":[249],"at":[250],"https://github.com/manestay/dogmatiq.":[251]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-08T00:00:00"}