{"id":"https://openalex.org/W3175350702","doi":null,"title":"Assessing the Quality of Human-Generated Summaries with Weakly Supervised Learning.","display_name":"Assessing the Quality of Human-Generated Summaries with Weakly Supervised Learning.","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3175350702","mag":"3175350702"},"language":"en","primary_location":{"id":"pmh:oai:dspace.ut.ee:10062/93052","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/10062/93052","pdf_url":"https://dspace.ut.ee/bitstreams/8349d48b-109d-4ca7-a97e-4e515739c919/download","source":{"id":"https://openalex.org/S4306401412","display_name":"DSpace repository (University of Tartu)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I56085075","host_organization_name":"University of Tartu","host_organization_lineage":["https://openalex.org/I56085075"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://dspace.ut.ee/bitstreams/8349d48b-109d-4ca7-a97e-4e515739c919/download","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5090576976","display_name":"Joakim Olsen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Joakim Olsen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063383258","display_name":"Arild Brandrud N\u00e6ss","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Arild Brandrud N\u00e6ss","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5008574874","display_name":"Pierre Lison","orcid":"https://orcid.org/0000-0002-7649-0689","observed_orcids":["https://orcid.org/0000-0002-7649-0689"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pierre Lison","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-7649-0689","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0725,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.33601833,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":93},"biblio":{"volume":null,"issue":null,"first_page":"112","last_page":"123"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.5893999934196472,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.5893999934196472,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.3752000033855438,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.008100000210106373,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/weakly-supervised-learning","display_name":"weakly supervised learning","score":0.968999981880188},{"id":"https://openalex.org/keywords/document-embeddings","display_name":"document embeddings","score":0.6790000200271606},{"id":"https://openalex.org/keywords/cosine-similarity","display_name":"cosine similarity","score":0.718999981880188},{"id":"https://openalex.org/keywords/norwegian-language","display_name":"Norwegian language","score":0.47999998927116394}],"concepts":[{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6474260687828064},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4720032215118408},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39391815662384033},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3720739781856537},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.08612948656082153},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:dspace.ut.ee:10062/93052","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/10062/93052","pdf_url":"https://dspace.ut.ee/bitstreams/8349d48b-109d-4ca7-a97e-4e515739c919/download","source":{"id":"https://openalex.org/S4306401412","display_name":"DSpace repository (University of Tartu)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I56085075","host_organization_name":"University of Tartu","host_organization_lineage":["https://openalex.org/I56085075"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"},{"id":"mag:3175350702","doi":null,"is_oa":false,"landing_page_url":"https://aclanthology.org/2021.nodalida-main.12/","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:dspace.ut.ee:10062/93052","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/10062/93052","pdf_url":"https://dspace.ut.ee/bitstreams/8349d48b-109d-4ca7-a97e-4e515739c919/download","source":{"id":"https://openalex.org/S4306401412","display_name":"DSpace repository (University of Tartu)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I56085075","host_organization_name":"University of Tartu","host_organization_lineage":["https://openalex.org/I56085075"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3175350702.pdf","grobid_xml":"https://content.openalex.org/works/W3175350702.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1843891098","https://openalex.org/W1877481539","https://openalex.org/W1880262756","https://openalex.org/W1980776243","https://openalex.org/W2030265142","https://openalex.org/W2102065370","https://openalex.org/W2104411075","https://openalex.org/W2107598941","https://openalex.org/W2120481102","https://openalex.org/W2131744502","https://openalex.org/W2147152072","https://openalex.org/W2154652894","https://openalex.org/W2251206996","https://openalex.org/W2251260323","https://openalex.org/W2307381258","https://openalex.org/W2317879529","https://openalex.org/W2480068437","https://openalex.org/W2739351760","https://openalex.org/W2752395160","https://openalex.org/W2769041395","https://openalex.org/W2798542795","https://openalex.org/W2799029897","https://openalex.org/W2888041867","https://openalex.org/W2895604144","https://openalex.org/W2896457183","https://openalex.org/W2898734514","https://openalex.org/W2903440991","https://openalex.org/W2910458567","https://openalex.org/W2936695845","https://openalex.org/W2948636975","https://openalex.org/W2950813464","https://openalex.org/W2963341956","https://openalex.org/W2963687836","https://openalex.org/W2997918300","https://openalex.org/W3022333975","https://openalex.org/W3030163527","https://openalex.org/W3035050380","https://openalex.org/W3105816068"],"related_works":["https://openalex.org/W4306755369","https://openalex.org/W2400500583","https://openalex.org/W2971779293","https://openalex.org/W3008926737","https://openalex.org/W3093303279","https://openalex.org/W2406741635","https://openalex.org/W2855233750","https://openalex.org/W2990051739","https://openalex.org/W4393303437","https://openalex.org/W2079361112"],"abstract_inverted_index":{"This":[0],"paper":[1],"explores":[2],"how":[3],"to":[4,63,96,112,142,175],"automatically":[5,41],"measure":[6],"the":[7,32,82,86,91,104,108,114,132,145,154,166,187,192],"quality":[8,84,117,181],"of":[9,17,46,85,139,150,162,168,179,186,197],"human-generated":[10],"summaries,":[11],"based":[12],"on":[13],"a":[14,44,65,70,79,97,124,195],"Norwegian":[15],"corpus":[16,193],"real":[18,33,188],"estate":[19,34,189],"condition":[20],"reports":[21,35,190],"and":[22,36,53,72,77,93,102],"their":[23,37],"corresponding":[24],"summaries.The":[25],"proposed":[26],"approach":[27],"proceeds":[28],"in":[29,191],"two":[30,109],"steps.First,":[31],"associated":[38],"summaries":[39],"are":[40,60],"labelled":[42],"using":[43,55],"set":[45],"heuristic":[47],"rules":[48],"gathered":[49],"from":[50],"human":[51,172],"experts":[52],"aggregated":[54,58,133],"weak":[56,137,155],"supervision.The":[57],"labels":[59,134,157,170],"then":[61],"employed":[62],"learn":[64],"neural":[66,88],"model":[67,89,126],"that":[68,153,183],"takes":[69],"document":[71,92,110],"its":[73,94],"summary":[74,95,116,163,180,196],"as":[75],"inputs":[76],"outputs":[78],"score":[80],"reflecting":[81],"predicted":[83],"summary.The":[87],"maps":[90],"shared":[98],"\"summary":[99],"content":[100],"space\"":[101],"computes":[103],"cosine":[105],"similarity":[106],"between":[107],"embeddings":[111],"predict":[113,182],"final":[115],"score.The":[118],"best":[119,146],"performance":[120],"is":[121],"achieved":[122],"by":[123],"CNN-based":[125],"with":[127,171],"an":[128],"accuracy":[129],"(measured":[130],"against":[131],"obtained":[135],"via":[136],"supervision)":[138],"89.5%,":[140],"compared":[141],"72.6%":[143],"for":[144],"unsupervised":[147],"model.Manual":[148],"inspection":[149],"examples":[151],"indicate":[152],"supervision":[156],"do":[158],"capture":[159],"important":[160],"indicators":[161],"quality,":[164],"but":[165],"correlation":[167],"those":[169],"judgements":[173],"remains":[174],"be":[176],"validated.Our":[177],"models":[178],"approximately":[184],"30%":[185],"have":[194],"poor":[198],"quality.":[199]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2021-07-05T00:00:00"}