{"id":"https://openalex.org/W3011993973","doi":"https://doi.org/10.1145/3343413.3377996","title":"Contrasting Human Opinion of Non-factoid Question Answering with Automatic Evaluation","display_name":"Contrasting Human Opinion of Non-factoid Question Answering with Automatic Evaluation","publication_year":2020,"publication_date":"2020-03-12","ids":{"openalex":"https://openalex.org/W3011993973","doi":"https://doi.org/10.1145/3343413.3377996","mag":"3011993973"},"language":"en","primary_location":{"id":"doi:10.1145/3343413.3377996","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3343413.3377996","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3343413.3377996","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3343413.3377996","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071808662","display_name":"Tianbo Ji","orcid":"https://orcid.org/0000-0003-0143-6220"},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]}],"countries":["IE"],"is_corresponding":true,"raw_author_name":"Tianbo Ji","raw_affiliation_strings":["Dublin City University, Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"Dublin City University, Dublin, Ireland","institution_ids":["https://openalex.org/I42934936"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020500064","display_name":"Yvette Graham","orcid":"https://orcid.org/0000-0001-6741-4855"},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Yvette Graham","raw_affiliation_strings":["Dublin City University, Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"Dublin City University, Dublin, Ireland","institution_ids":["https://openalex.org/I42934936"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018481328","display_name":"Gareth J. F. Jones","orcid":"https://orcid.org/0000-0003-2923-8365"},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Gareth J.F. Jones","raw_affiliation_strings":["Dublin City University, Dublin, Ireland"],"affiliations":[{"raw_affiliation_string":"Dublin City University, Dublin, Ireland","institution_ids":["https://openalex.org/I42934936"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5071808662"],"corresponding_institution_ids":["https://openalex.org/I42934936"],"apc_list":null,"apc_paid":null,"fwci":0.1359,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.53078155,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"348","last_page":"352"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.8781405687332153},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8525304198265076},{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.81742262840271},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6748229265213013},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6535862684249878},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6218324899673462},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5920109748840332},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.5431612133979797},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5295501947402954},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.529133677482605},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5127435922622681},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.07690563797950745}],"concepts":[{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.8781405687332153},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8525304198265076},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.81742262840271},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6748229265213013},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6535862684249878},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6218324899673462},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5920109748840332},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.5431612133979797},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5295501947402954},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.529133677482605},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5127435922622681},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07690563797950745},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3343413.3377996","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3343413.3377996","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3343413.3377996","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3343413.3377996","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3343413.3377996","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3343413.3377996","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 Conference on Human Information Interaction and Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7699999809265137,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G228147372","display_name":null,"funder_award_id":"Grant 13/RC/2106","funder_id":"https://openalex.org/F4320320847","funder_display_name":"Science Foundation Ireland"},{"id":"https://openalex.org/G2563910032","display_name":null,"funder_award_id":"13/RC/2106","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G3993907298","display_name":null,"funder_award_id":"13/RC/2106","funder_id":"https://openalex.org/F4320320847","funder_display_name":"Science Foundation Ireland"}],"funders":[{"id":"https://openalex.org/F4320320847","display_name":"Science Foundation Ireland","ror":"https://ror.org/0271asj38"},{"id":"https://openalex.org/F4320320865","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81"},{"id":"https://openalex.org/F4320335322","display_name":"European Regional Development Fund","ror":"https://ror.org/00k4n6c32"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3011993973.pdf","grobid_xml":"https://content.openalex.org/works/W3011993973.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W1966519405","https://openalex.org/W2101105183","https://openalex.org/W2116492146","https://openalex.org/W2150824314","https://openalex.org/W2167007060","https://openalex.org/W2260677151","https://openalex.org/W2415755012","https://openalex.org/W2512924740","https://openalex.org/W2779681640","https://openalex.org/W2794365787","https://openalex.org/W2889453388","https://openalex.org/W2898454195","https://openalex.org/W2949190861","https://openalex.org/W2952103439","https://openalex.org/W2963919867","https://openalex.org/W3002072934","https://openalex.org/W4231667498","https://openalex.org/W4252316627"],"related_works":["https://openalex.org/W2366403280","https://openalex.org/W1495108544","https://openalex.org/W2091301346","https://openalex.org/W3148229873","https://openalex.org/W4389760904","https://openalex.org/W2150160875","https://openalex.org/W4214678372","https://openalex.org/W1605559518","https://openalex.org/W4214601164","https://openalex.org/W4284703357"],"abstract_inverted_index":{"Evaluation":[0],"in":[1],"non-factoid":[2],"question":[3,82],"answering":[4,83],"tasks":[5],"generally":[6],"takes":[7],"the":[8,32,53,63,77,100,115,141],"form":[9],"of":[10,12,23,56,62,69,88,117,135],"computation":[11],"automatic":[13,36,89],"metric":[14],"scores":[15,33],"for":[16,67,93,140],"systems":[17],"on":[18],"a":[19,123,132],"sample":[20],"test":[21],"set":[22],"questions":[24],"against":[25],"human-generated":[26],"reference":[27],"answers.":[28],"Conclusions":[29],"drawn":[30,102],"from":[31,52,103],"produced":[34],"by":[35,86],"metrics":[37,90,105],"inevitably":[38],"lead":[39],"to":[40,96,107,127],"important":[41,78],"decisions":[42],"about":[43,110],"future":[44],"directions.":[45],"Metrics":[46],"commonly":[47],"applied":[48],"include":[49],"ROUGE,":[50],"adopted":[51],"related":[54],"field":[55],"summarization,":[57],"BLEU":[58],"and":[59,126],"Meteor,":[60],"both":[61],"latter":[64],"originally":[65,91],"developed":[66,138],"evaluation":[68,137],"machine":[70,118],"translation.":[71],"In":[72],"this":[73,129],"paper,":[74],"we":[75],"pose":[76],"question,":[79,130],"given":[80],"that":[81],"is":[84],"evaluated":[85],"application":[87],"designed":[92],"other":[94],"tasks,":[95],"what":[97],"degree":[98],"do":[99],"conclusions":[101],"such":[104],"correspond":[106],"human":[108,136],"opinion":[109],"system-generated":[111],"answers?":[112],"We":[113],"take":[114],"task":[116,142],"reading":[119],"comprehension":[120],"(MRC)":[121],"as":[122],"case":[124],"study":[125],"address":[128],"provide":[131],"new":[133],"method":[134],"specifically":[139],"at":[143],"hand.":[144]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}