{"id":"https://openalex.org/W2534147738","doi":"https://doi.org/10.1145/2983323.2983694","title":"Optimizing Nugget Annotations with Active Learning","display_name":"Optimizing Nugget Annotations with Active Learning","publication_year":2016,"publication_date":"2016-10-24","ids":{"openalex":"https://openalex.org/W2534147738","doi":"https://doi.org/10.1145/2983323.2983694","mag":"2534147738"},"language":"en","primary_location":{"id":"doi:10.1145/2983323.2983694","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2983323.2983694","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM International on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080828705","display_name":"Gaurav Baruah","orcid":"https://orcid.org/0000-0002-4992-0524"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Gaurav Baruah","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100392954","display_name":"Haotian Zhang","orcid":"https://orcid.org/0000-0001-7786-389X"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Haotian Zhang","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061857595","display_name":"Rakesh Guttikonda","orcid":null},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Rakesh Guttikonda","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082997975","display_name":"Jimmy Lin","orcid":"https://orcid.org/0000-0002-0661-7189"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jimmy Lin","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049222552","display_name":"Mark D. Smucker","orcid":"https://orcid.org/0000-0003-4968-6405"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Mark D. Smucker","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028031104","display_name":"Olga Vechtomova","orcid":"https://orcid.org/0000-0001-7371-0837"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Olga Vechtomova","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I151746483"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2359","last_page":"2364"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.8903928995132446},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7648392915725708},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7352679371833801},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6457903385162354},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5153602957725525},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.5101123452186584},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.5091089606285095},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.5074579119682312},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4638884663581848},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45693153142929077},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4327854812145233},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.07450738549232483}],"concepts":[{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.8903928995132446},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7648392915725708},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7352679371833801},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6457903385162354},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5153602957725525},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.5101123452186584},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.5091089606285095},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.5074579119682312},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4638884663581848},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45693153142929077},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4327854812145233},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07450738549232483},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2983323.2983694","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2983323.2983694","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 25th ACM International on Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6499999761581421,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320309327","display_name":"Google","ror":"https://ror.org/00njsd438"},{"id":"https://openalex.org/F4320322676","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W135190683","https://openalex.org/W219090214","https://openalex.org/W1493399377","https://openalex.org/W1523019309","https://openalex.org/W1700671489","https://openalex.org/W1977570008","https://openalex.org/W1986827153","https://openalex.org/W1988613011","https://openalex.org/W1989220366","https://openalex.org/W2017633929","https://openalex.org/W2029075138","https://openalex.org/W2037140704","https://openalex.org/W2066703653","https://openalex.org/W2096623622","https://openalex.org/W2100520226","https://openalex.org/W2138090731","https://openalex.org/W2145410165","https://openalex.org/W2185884126","https://openalex.org/W2341384651","https://openalex.org/W2404416613","https://openalex.org/W2604321711","https://openalex.org/W2918053736"],"related_works":["https://openalex.org/W2366403280","https://openalex.org/W1495108544","https://openalex.org/W2091301346","https://openalex.org/W3148229873","https://openalex.org/W4389760904","https://openalex.org/W2150160875","https://openalex.org/W2358294942","https://openalex.org/W2241813887","https://openalex.org/W2405355225","https://openalex.org/W4367460280"],"abstract_inverted_index":{"Nugget-based":[0],"evaluations,":[1],"such":[2],"as":[3,34],"those":[4],"deployed":[5,119],"in":[6,25,52,120],"the":[7,50,64,67,76],"TREC":[8,97],"Temporal":[9],"Summarization":[10],"and":[11,86],"Question":[12],"Answering":[13],"tracks,":[14],"require":[15],"human":[16],"assessors":[17],"to":[18,59,83],"determine":[19],"whether":[20],"a":[21,26,70,109],"nugget":[22,35,79],"is":[23,37,81],"present":[24,43],"given":[27,110],"piece":[28],"of":[29,112],"text.":[30],"This":[31],"process,":[32],"known":[33],"annotation,":[36],"labor-intensive.":[38],"In":[39],"this":[40],"paper,":[41],"we":[42,87],"two":[44],"active":[45],"learning":[46],"techniques":[47,103],"that":[48,66,78,101,116],"prioritize":[49],"sequence":[51],"which":[53],"candidate":[54],"nugget/sentence":[55],"pairs":[56],"are":[57,117],"presented":[58],"an":[60],"assessor,":[61],"based":[62],"on":[63,75],"likelihood":[65],"sentence":[68],"contains":[69],"nugget.":[71],"Our":[72],"approach":[73],"builds":[74],"recognition":[77],"annotation":[80],"similar":[82],"high-recall":[84],"retrieval,":[85],"adapt":[88],"proven":[89],"existing":[90,96],"solutions.":[91],"Simulation":[92],"experiments":[93],"with":[94],"four":[95],"test":[98],"collections":[99],"show":[100],"our":[102],"yield":[104],"far":[105],"more":[106],"matches":[107],"for":[108],"level":[111],"effort":[113],"than":[114],"baselines":[115],"typically":[118],"previous":[121],"nugget-based":[122],"evaluations.":[123]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}