{"id":"https://openalex.org/W2117763124","doi":"https://doi.org/10.21236/ada440382","title":"Reducing Labeling Effort for Structured Prediction Tasks","display_name":"Reducing Labeling Effort for Structured Prediction Tasks","alternate_titles":[],"publication_year":2005,"publication_date":"2005-01-01","ids":{"openalex":"https://openalex.org/W2117763124","doi":"https://doi.org/10.21236/ada440382","mag":"2117763124"},"language":"en","primary_location":{"id":"doi:10.21236/ada440382","doi":"https://doi.org/10.21236/ada440382","is_oa":false,"landing_page_url":"https://doi.org/10.21236/ada440382","pdf_url":null,"source":{"id":"https://openalex.org/S4406923043","display_name":"Defense Technical Information Center (DTIC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"OSD or Non-Service DoD Agency","raw_type":"report"},"parent_works":[],"lineage":["https://openalex.org/W2117763124"],"child_works_count":0,"type":"report","is_traditional":false,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011178134","display_name":"Aron Culotta","orcid":"https://orcid.org/0000-0003-2660-7575","observed_orcids":["https://orcid.org/0000-0003-2660-7575"]},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aron Culotta","raw_affiliation_strings":["Dept of Computer Science, University of Massachusetts, Amherst, MA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept of Computer Science, University of Massachusetts, Amherst, MA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5107835063","display_name":"Andrew McCallum","orcid":"https://orcid.org/0000-0003-2843-6992","observed_orcids":["https://orcid.org/0000-0003-2843-6992"]},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew McCallum","raw_affiliation_strings":["Dept of Computer Science, University of Massachusetts, Amherst, MA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept of Computer Science, University of Massachusetts, Amherst, MA","institution_ids":["https://openalex.org/I24603500"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":397,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/active-learning-machine-learning","display_name":"active learning (machine learning)","score":0.878000020980835},{"id":"https://openalex.org/keywords/structured-prediction","display_name":"structured prediction","score":0.9890000224113464},{"id":"https://openalex.org/keywords/annotation-effort","display_name":"annotation effort","score":0.5630000233650208},{"id":"https://openalex.org/keywords/labeling-cost","display_name":"labeling cost","score":0.4300000071525574},{"id":"https://openalex.org/keywords/user-annotation","display_name":"user annotation","score":0.38999998569488525}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5974177122116089},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32424938678741455}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.21236/ada440382","doi":"https://doi.org/10.21236/ada440382","is_oa":false,"landing_page_url":"https://doi.org/10.21236/ada440382","pdf_url":null,"source":{"id":"https://openalex.org/S4406923043","display_name":"Defense Technical Information Center (DTIC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"OSD or Non-Service DoD Agency","raw_type":"report"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.152.5914","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.152.5914","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://ciir.cs.umass.edu/pubfiles/ir-414.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.219.1846","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.219.1846","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.cs.umass.edu/~mccallum/papers/multichoice-aaai05.pdf","raw_type":"text"},{"id":"pmh:ADA440382","doi":null,"is_oa":false,"landing_page_url":"http://oai.dtic.mil/oai/oai?&amp;verb=getRecord&amp;metadataPrefix=html&amp;identifier=ADA440382","pdf_url":null,"source":{"id":"https://openalex.org/S4406923043","display_name":"Defense Technical Information Center (DTIC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"DTIC AND NTIS","raw_type":"Text"},{"id":"pmh:oai:works.bepress.com:andrew_mccallum-1265","doi":null,"is_oa":false,"landing_page_url":"https://works.bepress.com/andrew_mccallum/145","pdf_url":"https://works.bepress.com/andrew_mccallum/145/download/","source":{"id":"https://openalex.org/S4306402240","display_name":"ScholarWorks@UMassAmherst (University of Massachusetts Amherst)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I24603500","host_organization_name":"University of Massachusetts Amherst","host_organization_lineage":["https://openalex.org/I24603500"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Andrew McCallum","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G730876220","display_name":null,"funder_award_id":"NBCHD-03-0010","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G7704502625","display_name":"ITR:     Unified Graphical Models of Information Extraction and Data Mining with Application to Social Network Analysis","funder_award_id":"0326249","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320311089","display_name":"National Security Agency","ror":"https://ror.org/0047bvr32"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2117763124.pdf","grobid_xml":"https://content.openalex.org/works/W2117763124.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W191253314","https://openalex.org/W1513874326","https://openalex.org/W1524975733","https://openalex.org/W1580375566","https://openalex.org/W1945700406","https://openalex.org/W1980925709","https://openalex.org/W2004156982","https://openalex.org/W2114663556","https://openalex.org/W2125838338","https://openalex.org/W2147880316","https://openalex.org/W2151023586","https://openalex.org/W2163844356","https://openalex.org/W2949071206"],"related_works":["https://openalex.org/W4389524548","https://openalex.org/W2623634461","https://openalex.org/W1987605949","https://openalex.org/W2119720396","https://openalex.org/W1693085440","https://openalex.org/W2082351087","https://openalex.org/W7097879152","https://openalex.org/W7096618227","https://openalex.org/W6869908","https://openalex.org/W2165968176"],"abstract_inverted_index":{"A":[0],"common":[1],"obstacle":[2],"preventing":[3],"the":[4,13,46,53,63,67,81,97,130,156,171],"rapid":[5],"deployment":[6],"of":[7,15,39,50,83,174],"supervised":[8],"machine":[9],"learning":[10,57,73,121,162],"algorithms":[11],"is":[12,20,140],"lack":[14],"labeled":[16,70],"training":[17,31],"data.":[18],"This":[19],"particularly":[21],"expensive":[22],"to":[23,48,52,71,108,141,151],"obtain":[24],"for":[25,45,110,114],"structured":[26,92],"prediction":[27,93],"tasks,":[28],"where":[29],"each":[30,85,138],"instance":[32,47,139],"may":[33,105],"have":[34],"multiple,":[35],"interacting":[36],"labels,":[37],"all":[38],"which":[40,66,87,123],"must":[41,132],"be":[42,49,106],"correctly":[43],"annotated":[44],"use":[51],"learner.":[54],"Traditional":[55],"active":[56,120,161],"addresses":[58],"this":[59,76,160],"problem":[60],"by":[61,100,177],"optimizing":[62],"order":[64],"in":[65,91,164],"examples":[68],"are":[69],"increase":[72],"efficiency.":[74],"However,":[75],"approach":[77],"does":[78],"not":[79,125],"consider":[80],"difficulty":[82],"labeling":[84,98],"example,":[86,96],"can":[88],"vary":[89],"widely":[90],"tasks.":[94],"For":[95],"predicted":[99],"a":[101,118],"partially":[102,148],"trained":[103],"system":[104,144],"easier":[107],"correct":[109,149],"some":[111],"instances":[112,129],"than":[113],"others.":[115],"We":[116,158],"propose":[117],"new":[119],"paradigm":[122],"reduces":[124],"only":[126],"how":[127,136],"many":[128],"annotator":[131],"label,":[133],"but":[134],"also":[135],"difficult":[137],"annotate.":[142],"The":[143],"leverages":[145],"information":[146,167],"from":[147,155],"predictions":[150],"efficiently":[152],"solicit":[153],"annotations":[154],"user.":[157],"validate":[159],"framework":[163],"an":[165],"interactive":[166],"extraction":[168],"system,":[169],"reducing":[170],"total":[172],"number":[173],"annotation":[175],"actions":[176],"22%.":[178]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2117763124","counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":24},{"year":2024,"cited_by_count":22},{"year":2023,"cited_by_count":42},{"year":2022,"cited_by_count":35},{"year":2021,"cited_by_count":47},{"year":2020,"cited_by_count":37},{"year":2019,"cited_by_count":19},{"year":2018,"cited_by_count":24},{"year":2017,"cited_by_count":21},{"year":2016,"cited_by_count":17},{"year":2015,"cited_by_count":20},{"year":2014,"cited_by_count":20},{"year":2013,"cited_by_count":13},{"year":2012,"cited_by_count":7}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2016-06-24T00:00:00"}
