{"id":"https://openalex.org/W1966280106","doi":"https://doi.org/10.1007/s10579-013-9258-8","title":"Evaluating machine-assisted annotation in under-resourced settings","display_name":"Evaluating machine-assisted annotation in under-resourced settings","publication_year":2013,"publication_date":"2013-11-14","ids":{"openalex":"https://openalex.org/W1966280106","doi":"https://doi.org/10.1007/s10579-013-9258-8","mag":"1966280106"},"language":"en","primary_location":{"id":"doi:10.1007/s10579-013-9258-8","is_oa":false,"landing_page_url":"https://doi.org/10.1007/s10579-013-9258-8","pdf_url":null,"source":{"id":"https://openalex.org/S4306424877","display_name":"Language Resources and Evaluation","issn_l":"1574-020X","issn":["1574-020X","1574-0218"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Language Resources and Evaluation","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060115458","display_name":"Paul Felt","orcid":null},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Paul Felt","raw_affiliation_strings":["Department of Computer Science, Brigham Young University, Provo, UT, USA","Department of Computer Science, Brigham Young University, Provo, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]},{"raw_affiliation_string":"Department of Computer Science, Brigham Young University, Provo, USA#TAB#","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090791372","display_name":"Eric K. Ringger","orcid":null},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eric K. Ringger","raw_affiliation_strings":["Department of Computer Science, Brigham Young University, Provo, UT, USA","Department of Computer Science, Brigham Young University, Provo, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]},{"raw_affiliation_string":"Department of Computer Science, Brigham Young University, Provo, USA#TAB#","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029074659","display_name":"Kevin Seppi","orcid":"https://orcid.org/0000-0002-4932-2700"},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Seppi","raw_affiliation_strings":["Department of Computer Science, Brigham Young University, Provo, UT, USA","Department of Computer Science, Brigham Young University, Provo, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]},{"raw_affiliation_string":"Department of Computer Science, Brigham Young University, Provo, USA#TAB#","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061488216","display_name":"Kristian S. Heal","orcid":"https://orcid.org/0000-0002-7710-5252"},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kristian S. Heal","raw_affiliation_strings":["Neal A. Maxwell Institute for Religious Scholarship, Center for the Preservation of Ancient Religious Texts, Brigham Young University, Provo, UT, USA","Neal A. Maxwell Institute for Religious Scholarship, Center for the Preservation of Ancient Religious Texts, Brigham Young University, Provo, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Neal A. Maxwell Institute for Religious Scholarship, Center for the Preservation of Ancient Religious Texts, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]},{"raw_affiliation_string":"Neal A. Maxwell Institute for Religious Scholarship, Center for the Preservation of Ancient Religious Texts, Brigham Young University, Provo, USA","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017536910","display_name":"Robbie Haertel","orcid":null},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robbie A. Haertel","raw_affiliation_strings":["Department of Computer Science, Brigham Young University, Provo, UT, USA","Department of Computer Science, Brigham Young University, Provo, USA#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]},{"raw_affiliation_string":"Department of Computer Science, Brigham Young University, Provo, USA#TAB#","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017104888","display_name":"Deryle Lonsdale","orcid":null},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deryle Lonsdale","raw_affiliation_strings":["Department of Linguistics and English Language, Brigham Young University, Provo, UT, USA","Department of Linguistics and English Language, Brigham Young University, Provo, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Linguistics and English Language, Brigham Young University, Provo, UT, USA","institution_ids":["https://openalex.org/I100005738"]},{"raw_affiliation_string":"Department of Linguistics and English Language, Brigham Young University, Provo, USA","institution_ids":["https://openalex.org/I100005738"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5060115458"],"corresponding_institution_ids":["https://openalex.org/I100005738"],"apc_list":{"value":3090,"currency":"USD","value_usd":3090},"apc_paid":null,"fwci":2.2365,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.89435217,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":"48","issue":"4","first_page":"561","last_page":"599"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8554500937461853},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8028548955917358},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.626929759979248},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6050832271575928},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5388900637626648},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5229516625404358},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.49941492080688477},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.49454113841056824},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4364752173423767},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.35970520973205566}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8554500937461853},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8028548955917358},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.626929759979248},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6050832271575928},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5388900637626648},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5229516625404358},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49941492080688477},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.49454113841056824},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4364752173423767},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.35970520973205566},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s10579-013-9258-8","is_oa":false,"landing_page_url":"https://doi.org/10.1007/s10579-013-9258-8","pdf_url":null,"source":{"id":"https://openalex.org/S4306424877","display_name":"Language Resources and Evaluation","issn_l":"1574-020X","issn":["1574-020X","1574-0218"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Language Resources and Evaluation","raw_type":"journal-article"},{"id":"pmh:oai:scholarsarchive.byu.edu:facpub-7723","is_oa":false,"landing_page_url":"https://link.springer.com/article/10.1007/s10579-013-9258-8","pdf_url":null,"source":{"id":"https://openalex.org/S4377196308","display_name":"ScholarsArchive  (Brigham Young University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I100005738","host_organization_name":"Brigham Young University","host_organization_lineage":["https://openalex.org/I100005738"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Faculty Publications","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":69,"referenced_works":["https://openalex.org/W7688137","https://openalex.org/W136468075","https://openalex.org/W153277182","https://openalex.org/W155589836","https://openalex.org/W360824148","https://openalex.org/W636164561","https://openalex.org/W1496189301","https://openalex.org/W1504251463","https://openalex.org/W1505364729","https://openalex.org/W1528941926","https://openalex.org/W1574901103","https://openalex.org/W1632114991","https://openalex.org/W1809801995","https://openalex.org/W1894835849","https://openalex.org/W1979327299","https://openalex.org/W2019096529","https://openalex.org/W2028149440","https://openalex.org/W2035683813","https://openalex.org/W2036242127","https://openalex.org/W2044688197","https://openalex.org/W2045656233","https://openalex.org/W2067349145","https://openalex.org/W2088490356","https://openalex.org/W2095729244","https://openalex.org/W2096806755","https://openalex.org/W2097726431","https://openalex.org/W2098949613","https://openalex.org/W2105897558","https://openalex.org/W2109439962","https://openalex.org/W2110308599","https://openalex.org/W2117729721","https://openalex.org/W2125268123","https://openalex.org/W2125327503","https://openalex.org/W2133993935","https://openalex.org/W2137228101","https://openalex.org/W2137766371","https://openalex.org/W2143413399","https://openalex.org/W2147468270","https://openalex.org/W2148400971","https://openalex.org/W2151401338","https://openalex.org/W2157887077","https://openalex.org/W2161002933","https://openalex.org/W2163844356","https://openalex.org/W2164455818","https://openalex.org/W2166496241","https://openalex.org/W2166706824","https://openalex.org/W2169336678","https://openalex.org/W2171278097","https://openalex.org/W2251975282","https://openalex.org/W2334652388","https://openalex.org/W2341078838","https://openalex.org/W2903158431","https://openalex.org/W2911253603","https://openalex.org/W2914605928","https://openalex.org/W3089160002","https://openalex.org/W4205184193","https://openalex.org/W4211120398","https://openalex.org/W4232383088","https://openalex.org/W4241850027","https://openalex.org/W4285719527","https://openalex.org/W6606205340","https://openalex.org/W6606292167","https://openalex.org/W6612108521","https://openalex.org/W6630313444","https://openalex.org/W6631387317","https://openalex.org/W6659517344","https://openalex.org/W6720379491","https://openalex.org/W6756615331","https://openalex.org/W6758179512"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328","https://openalex.org/W1533177136","https://openalex.org/W2251519152","https://openalex.org/W4380994516"],"abstract_inverted_index":null,"counts_by_year":[{"year":2019,"cited_by_count":1},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":2}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}