{"id":"https://openalex.org/W7203946535","doi":"https://doi.org/10.48550/arxiv.2608.18888","title":"Assessing Quality of Experience in Natural Language Generation of German Text","display_name":"Assessing Quality of Experience in Natural Language Generation of German Text","publication_year":2026,"publication_date":"2026-08-19","ids":{"openalex":"https://openalex.org/W7203946535","doi":"https://doi.org/10.48550/arxiv.2608.18888"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.18888","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18888","pdf_url":"https://arxiv.org/pdf/2608.18888","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.18888","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050070871","display_name":"Dinh Nam Pham","orcid":"https://orcid.org/0000-0002-9431-5614","observed_orcids":["https://orcid.org/0000-0002-9431-5614"]},"institutions":[{"id":"https://openalex.org/I33256026","display_name":"German Research Centre for Artificial Intelligence","ror":"https://ror.org/01ayc5b57","country_code":"DE","type":"funder","lineage":["https://openalex.org/I33256026"]},{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Pham, Dinh Nam","raw_affiliation_strings":["Speech and Language Technology Lab , German Research Center for Artificial Intelligence (DFKI) , Berlin , Germany","Technische Universit\u00e4t Berlin , Berlin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Speech and Language Technology Lab , German Research Center for Artificial Intelligence (DFKI) , Berlin , Germany","institution_ids":["https://openalex.org/I33256026"]},{"raw_affiliation_string":"Technische Universit\u00e4t Berlin , Berlin , Germany","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093458315","display_name":"Shushen Manakhimova","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I33256026","display_name":"German Research Centre for Artificial Intelligence","ror":"https://ror.org/01ayc5b57","country_code":"DE","type":"funder","lineage":["https://openalex.org/I33256026"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Manakhimova, Shushen","raw_affiliation_strings":["Speech and Language Technology Lab , German Research Center for Artificial Intelligence (DFKI) , Berlin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Speech and Language Technology Lab , German Research Center for Artificial Intelligence (DFKI) , Berlin , Germany","institution_ids":["https://openalex.org/I33256026"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088185761","display_name":"Vivien Macketanz","orcid":"https://orcid.org/0000-0003-4530-6241","observed_orcids":["https://orcid.org/0000-0003-4530-6241"]},"institutions":[{"id":"https://openalex.org/I33256026","display_name":"German Research Centre for Artificial Intelligence","ror":"https://ror.org/01ayc5b57","country_code":"DE","type":"funder","lineage":["https://openalex.org/I33256026"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Macketanz, Vivien","raw_affiliation_strings":["Speech and Language Technology Lab , German Research Center for Artificial Intelligence (DFKI) , Berlin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Speech and Language Technology Lab , German Research Center for Artificial Intelligence (DFKI) , Berlin , Germany","institution_ids":["https://openalex.org/I33256026"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075001284","display_name":"Sebastian M\u00f6ller","orcid":"https://orcid.org/0000-0003-3057-0760","observed_orcids":["https://orcid.org/0000-0003-3057-0760"]},"institutions":[{"id":"https://openalex.org/I33256026","display_name":"German Research Centre for Artificial Intelligence","ror":"https://ror.org/01ayc5b57","country_code":"DE","type":"funder","lineage":["https://openalex.org/I33256026"]},{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"M\u00f6ller, Sebastian","raw_affiliation_strings":["Speech and Language Technology Lab , German Research Center for Artificial Intelligence (DFKI) , Berlin , Germany","Technische Universit\u00e4t Berlin , Berlin , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Speech and Language Technology Lab , German Research Center for Artificial Intelligence (DFKI) , Berlin , Germany","institution_ids":["https://openalex.org/I33256026"]},{"raw_affiliation_string":"Technische Universit\u00e4t Berlin , Berlin , Germany","institution_ids":["https://openalex.org/I4577782"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.7914999723434448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.7914999723434448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11649999767541885,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.01759999990463257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-language-generation-evaluation","display_name":"natural language generation evaluation","score":0.859000027179718},{"id":"https://openalex.org/keywords/quality-of-experience","display_name":"quality of experience","score":0.9860000014305115},{"id":"https://openalex.org/keywords/human-centered-evaluation","display_name":"human-centered evaluation","score":0.7900000214576721},{"id":"https://openalex.org/keywords/text-summarization","display_name":"text summarization","score":0.6890000104904175},{"id":"https://openalex.org/keywords/machine-translation","display_name":"machine translation","score":0.9620000123977661},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.6050000190734863},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7799999713897705}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.79339998960495},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.7498000264167786},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.6955999732017517},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6851999759674072},{"id":"https://openalex.org/C154775046","wikidata":"https://www.wikidata.org/wiki/Q188","display_name":"German","level":2,"score":0.6151000261306763},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5699999928474426},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5573999881744385},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5299000144004822},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4830999970436096},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.43369999527931213},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.4269999861717224},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3822000026702881},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3621000051498413},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3402999937534332},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.3352000117301941},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.3237000107765198},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2897999882698059},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.2856999933719635},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.26339998841285706},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.25600001215934753}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.18888","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18888","pdf_url":"https://arxiv.org/pdf/2608.18888","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.18888","doi":"https://doi.org/10.48550/arxiv.2608.18888","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.18888","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.18888","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18888","pdf_url":"https://arxiv.org/pdf/2608.18888","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality education","score":0.5262954831123352}],"study_designs":[],"awards":[{"id":"https://openalex.org/G7352007896","display_name":"Analyse und automatische Absch\u00e4tzung der Qualit\u00e4t maschinell generierter Texte","funder_award_id":"436813723","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203946535.pdf","grobid_xml":"https://content.openalex.org/works/W7203946535.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2745039414","https://openalex.org/W4378876515","https://openalex.org/W4389519254","https://openalex.org/W3037186790","https://openalex.org/W2979827194","https://openalex.org/W4397028790","https://openalex.org/W3033016100","https://openalex.org/W1590378318","https://openalex.org/W4417307924","https://openalex.org/W1156568792"],"abstract_inverted_index":{"The":[0,133],"rapid":[1],"advancement":[2],"of":[3,13,44,60,66,129],"Natural":[4],"Language":[5],"Generation":[6],"(NLG)":[7],"has":[8],"made":[9],"the":[10,41,127],"reliable":[11],"evaluation":[12,59,164],"generated":[14],"text":[15,72],"increasingly":[16],"critical,":[17],"as":[18,22],"these":[19],"systems,":[20],"such":[21],"large":[23],"language":[24,131],"models":[25,111],"(LLMs),":[26],"are":[27],"now":[28],"widely":[29],"deployed":[30],"in":[31,116],"real-world":[32],"applications.":[33],"However,":[34],"traditional":[35],"automatic":[36,71,99,168],"metrics":[37],"fail":[38],"to":[39,152],"capture":[40],"multifaceted":[42],"nature":[43],"perceived":[45],"quality.":[46],"In":[47],"this":[48],"paper,":[49],"we":[50,83],"introduce":[51],"TextQ-German,":[52],"a":[53,64,158,172],"novel":[54],"dataset":[55,134],"suite":[56],"for":[57,94,162,167,174],"human-centered":[58],"German":[61,81],"NLG":[62,163,176],"from":[63],"Quality":[65],"Experience":[67],"(QoE)":[68],"perspective,":[69],"covering":[70],"summarization":[73],"and":[74,88,107,165],"machine":[75],"translation.":[76],"Through":[77],"crowdsourcing":[78],"studies":[79],"with":[80,137,141,181],"speakers,":[82],"collect":[84],"human":[85,182],"quality":[86,92,183],"ratings":[87],"identify":[89],"relevant":[90],"perceptual":[91],"dimensions":[93],"each":[95],"task.":[96],"We":[97],"develop":[98],"QoE":[100,143,169],"prediction":[101],"models,":[102],"including":[103],"transformer-based,":[104],"linguistic":[105,122],"feature-based,":[106],"hybrid":[108],"approaches.":[109],"Hybrid":[110],"outperform":[112],"pure":[113],"transformer":[114],"baselines":[115,166],"almost":[117],"all":[118],"experimental":[119],"settings,":[120],"while":[121],"features":[123],"alone":[124],"can":[125],"approach":[126],"performance":[128],"fine-tuned":[130],"models.":[132],"is":[135],"extended":[136],"LLM-generated":[138],"outputs":[139],"annotated":[140],"overall":[142],"scores.":[144],"Final":[145],"validation":[146],"on":[147],"held-out":[148],"sets":[149],"indicates":[150],"generalization":[151],"unseen":[153],"data.":[154],"Our":[155],"work":[156],"contributes":[157],"publicly":[159],"accessible":[160],"resource":[161],"prediction,":[170],"providing":[171],"foundation":[173],"developing":[175],"systems":[177],"that":[178],"better":[179],"align":[180],"perception.":[184]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-08-22T00:00:00"}