{"id":"https://openalex.org/W82604178","doi":null,"title":"Impact of Annotation Difficulty on Automatically Detecting Problem Localization of Peer-Review Feedback","display_name":"Impact of Annotation Difficulty on Automatically Detecting Problem Localization of Peer-Review Feedback","publication_year":2010,"publication_date":"2010-01-01","ids":{"openalex":"https://openalex.org/W82604178","mag":"82604178"},"language":"en","primary_location":{"id":"pmh:oai:d-scholarship.pitt.edu:22952","doi":null,"is_oa":true,"landing_page_url":"https://d-scholarship.pitt.edu/22952/8/licence.txt","pdf_url":"http://d-scholarship.pitt.edu/22952/1/LitmanSchunnImpactOfAnnotation.pdf","source":{"id":"https://openalex.org/S4306402372","display_name":"D-Scholarship@Pitt (University of Pittsburgh)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I170201317","host_organization_name":"University of Pittsburgh","host_organization_lineage":["https://openalex.org/I170201317"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference or Workshop Item"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://d-scholarship.pitt.edu/22952/1/LitmanSchunnImpactOfAnnotation.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070226310","display_name":"Wenting Xiong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wenting Xiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048682514","display_name":"Diane J. Litman","orcid":"https://orcid.org/0000-0001-7282-7531","observed_orcids":["https://orcid.org/0000-0001-7282-7531"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Diane Litman","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-7282-7531","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5016157521","display_name":"Christian Dieter Schunn","orcid":"https://orcid.org/0000-0003-3589-297X","observed_orcids":["https://orcid.org/0000-0003-3589-297X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Christian D. Schunn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":93},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.4668000042438507,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.4668000042438507,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10959","display_name":"Student Assessment and Feedback","score":0.10790000110864639,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.09650000184774399,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/peer-review-feedback","display_name":"peer review feedback","score":0.8870000243186951},{"id":"https://openalex.org/keywords/inter-annotator-agreement","display_name":"inter-annotator agreement","score":0.9150000214576721},{"id":"https://openalex.org/keywords/student-peer-review","display_name":"student peer review","score":0.39399999380111694},{"id":"https://openalex.org/keywords/linguistic-features","display_name":"linguistic features","score":0.7689999938011169},{"id":"https://openalex.org/keywords/rule-based-prediction","display_name":"rule-based prediction","score":0.32600000500679016},{"id":"https://openalex.org/keywords/training-data-quality","display_name":"training data quality","score":0.4580000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7686150074005127},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7328467965126038},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6019587516784668},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.6016616821289062},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.517444908618927},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4554448127746582},{"id":"https://openalex.org/C53839665","wikidata":"https://www.wikidata.org/wiki/Q2067088","display_name":"Peer assessment","level":2,"score":0.4263029396533966},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.10519707202911377},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:d-scholarship.pitt.edu:22952","doi":null,"is_oa":true,"landing_page_url":"https://d-scholarship.pitt.edu/22952/8/licence.txt","pdf_url":"http://d-scholarship.pitt.edu/22952/1/LitmanSchunnImpactOfAnnotation.pdf","source":{"id":"https://openalex.org/S4306402372","display_name":"D-Scholarship@Pitt (University of Pittsburgh)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I170201317","host_organization_name":"University of Pittsburgh","host_organization_lineage":["https://openalex.org/I170201317"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference or Workshop Item"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.307.4585","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.307.4585","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.lrdc.pitt.edu/pubs/Abstracts/LitmanSchunnImpactOfAnnotation.pdf","raw_type":"text"},{"id":"mag:82604178","doi":null,"is_oa":false,"landing_page_url":"http://d-scholarship.pitt.edu/22952/1/LitmanSchunnImpactOfAnnotation.pdf","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:d-scholarship.pitt.edu:22952","doi":null,"is_oa":true,"landing_page_url":"https://d-scholarship.pitt.edu/22952/8/licence.txt","pdf_url":"http://d-scholarship.pitt.edu/22952/1/LitmanSchunnImpactOfAnnotation.pdf","source":{"id":"https://openalex.org/S4306402372","display_name":"D-Scholarship@Pitt (University of Pittsburgh)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I170201317","host_organization_name":"University of Pittsburgh","host_organization_lineage":["https://openalex.org/I170201317"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference or Workshop Item"},"sustainable_development_goals":[{"score":0.5,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W82604178.pdf","grobid_xml":"https://content.openalex.org/works/W82604178.grobid-xml"},"referenced_works_count":8,"referenced_works":["https://openalex.org/W1493615237","https://openalex.org/W1501335461","https://openalex.org/W1514944708","https://openalex.org/W1882605238","https://openalex.org/W2030441548","https://openalex.org/W2075371609","https://openalex.org/W2115612165","https://openalex.org/W2131853730"],"related_works":["https://openalex.org/W1493615237","https://openalex.org/W3033547830","https://openalex.org/W1501335461","https://openalex.org/W299930192","https://openalex.org/W2109121591","https://openalex.org/W2571065774","https://openalex.org/W2597225647","https://openalex.org/W1486227774","https://openalex.org/W7095553946","https://openalex.org/W1816643718"],"abstract_inverted_index":{"Abstract.":[0],"We":[1,22],"believe":[2],"that":[3,88,139],"providing":[4],"assessment":[5],"on":[6,24,91,152],"students":[7,13],"\u2019":[8],"reviewing":[9],"performance":[10,188],"will":[11],"enable":[12],"to":[14,53,74],"improve":[15],"the":[16,30,39,62,70,98,101,104,113,155,186,192],"quality":[17],"of":[18,29,43,61,72,103,115,126,199,210],"their":[19],"peer":[20,36,211],"reviews.":[21],"focus":[23],"assessing":[25],"one":[26],"particular":[27],"aspect":[28],"textual":[31,84],"feedback":[32,46,116,148,178],"contained":[33],"in":[34,65,205],"a":[35],"review":[37],"\u2013":[38],"presence":[40],"or":[41],"absence":[42],"problem":[44,48,76,93,119,153,200,203],"localization;":[45],"containing":[47],"localization":[49,77,94,120,204],"has":[50],"been":[51],"shown":[52],"be":[54],"associated":[55],"with":[56,123,159],"increased":[57],"understanding":[58],"and":[59,100,132,144,162,175,184],"implementation":[60],"feedback.":[63],"While":[64],"prior":[66],"work":[67],"we":[68,86,111],"demonstrated":[69],"feasibility":[71],"learning":[73],"predict":[75],"using":[78,146,177],"linguistic":[79,169],"features":[80],"automatically":[81],"extracted":[82],"from":[83],"feedback,":[85],"hypothesize":[87],"inter-annotator":[89,213],"disagreement":[90],"labeling":[92],"might":[95],"impact":[96],"both":[97,130,157,182],"accuracy":[99],"content":[102],"predictive":[105],"models.":[106,135],"To":[107],"test":[108],"this":[109],"hypothesis,":[110],"compare":[112],"use":[114],"examples":[117,179],"where":[118,149,180],"is":[121],"labeled":[122],"differing":[124],"levels":[125],"annotator":[127],"agreement,":[128,214],"for":[129],"training":[131,174],"testing":[133,176],"our":[134],"Our":[136],"results":[137],"show":[138],"when":[140,173],"models":[141,156],"are":[142],"trained":[143],"tested":[145],"only":[147],"annotators":[150,181],"agree":[151,183],"localization,":[154],"perform":[158],"high":[160],"accuracy,":[161],"contain":[163],"rules":[164,194],"involving":[165],"just":[166],"two":[167],"simple":[168],"features.":[170],"In":[171],"contrast,":[172],"disagree,":[185],"model":[187],"slightly":[189],"drops,":[190],"but":[191],"learned":[193],"capture":[195],"more":[196],"subtle":[197],"patterns":[198],"localization.":[201],"Keywords":[202],"text":[206],"comments,":[207],"data":[208],"mining":[209],"reviews,":[212],"natural":[215],"language":[216],"1":[217]},"counts_by_year":[{"year":2014,"cited_by_count":1}],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2016-06-24T00:00:00"}