{"id":"https://openalex.org/W7160406218","doi":"https://doi.org/10.48550/arxiv.2605.03624","title":"Annotation Quality in Aspect-Based Sentiment Analysis: A Case Study Comparing Experts, Students, Crowdworkers, and Large Language Model","display_name":"Annotation Quality in Aspect-Based Sentiment Analysis: A Case Study Comparing Experts, Students, Crowdworkers, and Large Language Model","publication_year":2026,"publication_date":"2026-05-05","ids":{"openalex":"https://openalex.org/W7160406218","doi":"https://doi.org/10.48550/arxiv.2605.03624"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.03624","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.03624","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.03624","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135477036","display_name":"Niklas Donhauser","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Donhauser, Niklas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088698198","display_name":"Jakob Fehle","orcid":"https://orcid.org/0009-0001-7845-9511"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fehle, Jakob","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093323666","display_name":"Nils Constantin Hellwig","orcid":"https://orcid.org/0009-0000-7305-8797"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hellwig, Nils Constantin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135430263","display_name":"Markus Weinberger","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weinberger, Markus","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014534985","display_name":"Udo Kruschwitz","orcid":"https://orcid.org/0000-0002-5503-0341"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kruschwitz, Udo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135536304","display_name":"Christian Wolff","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wolff, Christian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9872999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.001500000013038516,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8651999831199646},{"id":"https://openalex.org/keywords/german","display_name":"German","score":0.7353000044822693},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"Sentiment analysis","score":0.6115000247955322},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.59170001745224},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5608000159263611},{"id":"https://openalex.org/keywords/temporal-annotation","display_name":"Temporal annotation","score":0.42010000348091125}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8651999831199646},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8015000224113464},{"id":"https://openalex.org/C154775046","wikidata":"https://www.wikidata.org/wiki/Q188","display_name":"German","level":2,"score":0.7353000044822693},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6636000275611877},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6183000206947327},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.6115000247955322},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.59170001745224},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5608000159263611},{"id":"https://openalex.org/C7044111","wikidata":"https://www.wikidata.org/wiki/Q15844891","display_name":"Temporal annotation","level":5,"score":0.42010000348091125},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4043000042438507},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3695000112056732},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.36090001463890076},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3197000026702881},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30399999022483826},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.03624","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.03624","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.03624","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.03624","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5345107913017273,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Aspect-Based":[0],"Sentiment":[1,116,122],"Analysis":[2,117],"(ABSA)":[3],"enables":[4],"fine-grained":[5],"opinion":[6],"analysis":[7],"by":[8,66,82],"identifying":[9],"sentiments":[10],"toward":[11],"specific":[12],"aspects":[13],"or":[14],"targets":[15],"within":[16],"a":[17,70,76],"text.":[18],"While":[19],"ABSA":[20,108],"has":[21],"been":[22],"widely":[23],"studied":[24],"for":[25,78,106,130,164],"English,":[26],"research":[27],"on":[28,102,113],"other":[29],"languages":[30],"such":[31],"as":[32,75],"German":[33,56],"remains":[34],"limited,":[35],"largely":[36],"due":[37],"to":[38,68,138],"the":[39,53],"lack":[40],"of":[41,55],"high-quality":[42],"annotated":[43],"datasets.":[44],"This":[45],"paper":[46],"examines":[47],"how":[48],"different":[49,107],"annotation":[50,158],"sources":[51],"influence":[52],"development":[54],"ABSA.":[57],"To":[58],"this":[59],"end,":[60],"an":[61],"existing":[62],"dataset":[63,165],"is":[64,93],"re-annotated":[65],"experts":[67],"establish":[69],"ground":[71],"truth,":[72],"which":[73],"serves":[74],"reference":[77],"evaluating":[79],"annotations":[80],"produced":[81],"students,":[83],"crowdworkers,":[84],"Large":[85],"Language":[86,170],"Models":[87],"(LLMs),":[88],"and":[89,99,119,135,144,160],"experts.":[90],"Annotation":[91],"quality":[92],"compared":[94],"using":[95],"Inter-Annotator":[96],"Agreement":[97],"(IAA)":[98],"its":[100],"impact":[101],"downstream":[103],"model":[104],"performance":[105,140],"subtasks.":[109],"The":[110,150],"evaluation":[111],"focuses":[112],"Aspect":[114,121],"Category":[115],"(ACSA)":[118],"Target":[120],"Detection":[123],"(TASD).":[124],"We":[125],"apply":[126],"State-of-the-Art":[127],"(SOTA)":[128],"methods":[129],"ABSA,":[131],"including":[132],"BERT-,":[133],"T5-,":[134],"LLaMA-based":[136],"approaches":[137],"assess":[139],"differences,":[141],"spanning":[142],"fine-tuning":[143],"in-context":[145],"learning":[146],"with":[147],"instruction":[148],"prompts.":[149],"findings":[151],"provide":[152],"practical":[153],"insights":[154],"into":[155],"trade-offs":[156],"between":[157],"reliability":[159],"efficiency,":[161],"offering":[162],"guidance":[163],"construction":[166],"in":[167],"under-resourced":[168],"Natural":[169],"Processing":[171],"(NLP)":[172],"scenarios.":[173]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-07T00:00:00"}