{"id":"https://openalex.org/W4416576243","doi":"https://doi.org/10.1016/j.datak.2025.102534","title":"Human vs. Automated data annotation: Labeling the data set for an ML-driven support ticket classifier","display_name":"Human vs. Automated data annotation: Labeling the data set for an ML-driven support ticket classifier","publication_year":2025,"publication_date":"2025-11-23","ids":{"openalex":"https://openalex.org/W4416576243","doi":"https://doi.org/10.1016/j.datak.2025.102534"},"language":"en","primary_location":{"id":"doi:10.1016/j.datak.2025.102534","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.datak.2025.102534","pdf_url":null,"source":{"id":"https://openalex.org/S136993123","display_name":"Data & Knowledge Engineering","issn_l":"0169-023X","issn":["0169-023X","1872-6933"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data & Knowledge Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.datak.2025.102534","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086077369","display_name":"Simon Fuchs","orcid":"https://orcid.org/0000-0002-7372-4232"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Simon Fuchs","raw_affiliation_strings":["Technische Universit\u00e4t M\u00fcnchen, Boltzmannstra\u00dfe 3 Garching near Munich, 85748 , Bavaria, Germany"],"raw_orcid":"https://orcid.org/0000-0002-7372-4232","affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t M\u00fcnchen, Boltzmannstra\u00dfe 3 Garching near Munich, 85748 , Bavaria, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056226733","display_name":"Janik Schnellbach","orcid":null},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Janik Schnellbach","raw_affiliation_strings":["Technische Universit\u00e4t M\u00fcnchen, Boltzmannstra\u00dfe 3 Garching near Munich, 85748 , Bavaria, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t M\u00fcnchen, Boltzmannstra\u00dfe 3 Garching near Munich, 85748 , Bavaria, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003369792","display_name":"Holger Wittges","orcid":"https://orcid.org/0000-0002-3593-5290"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Holger Wittges","raw_affiliation_strings":["Technische Universit\u00e4t M\u00fcnchen, Boltzmannstra\u00dfe 3 Garching near Munich, 85748 , Bavaria, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t M\u00fcnchen, Boltzmannstra\u00dfe 3 Garching near Munich, 85748 , Bavaria, Germany","institution_ids":["https://openalex.org/I62916508"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077586272","display_name":"Helmut Krcmar","orcid":"https://orcid.org/0000-0002-2754-8493"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Helmut Krcmar","raw_affiliation_strings":["Technische Universit\u00e4t M\u00fcnchen, Boltzmannstra\u00dfe 3 Garching near Munich, 85748 , Bavaria, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t M\u00fcnchen, Boltzmannstra\u00dfe 3 Garching near Munich, 85748 , Bavaria, Germany","institution_ids":["https://openalex.org/I62916508"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5086077369"],"corresponding_institution_ids":["https://openalex.org/I62916508"],"apc_list":{"value":2590,"currency":"USD","value_usd":2590},"apc_paid":{"value":2590,"currency":"USD","value_usd":2590},"fwci":3.614,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.93924951,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"162","issue":null,"first_page":"102534","last_page":"102534"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.6308000087738037,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.6308000087738037,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.09130000323057175,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.04410000145435333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8465999960899353},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.6319000124931335},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.44909998774528503},{"id":"https://openalex.org/keywords/ticket","display_name":"Ticket","score":0.43950000405311584},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.43070000410079956},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4219000041484833},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.3806000053882599}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8465999960899353},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7789000272750854},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.6319000124931335},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5232999920845032},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5135999917984009},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.44909998774528503},{"id":"https://openalex.org/C2776540713","wikidata":"https://www.wikidata.org/wiki/Q7800647","display_name":"Ticket","level":2,"score":0.43950000405311584},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.43070000410079956},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4219000041484833},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.38760000467300415},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3806000053882599},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.37070000171661377},{"id":"https://openalex.org/C58973888","wikidata":"https://www.wikidata.org/wiki/Q1041418","display_name":"Semi-supervised learning","level":2,"score":0.3693999946117401},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.33869999647140503},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.3176000118255615},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.31709998846054077},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.2770000100135803}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.datak.2025.102534","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.datak.2025.102534","pdf_url":null,"source":{"id":"https://openalex.org/S136993123","display_name":"Data & Knowledge Engineering","issn_l":"0169-023X","issn":["0169-023X","1872-6933"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data & Knowledge Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1016/j.datak.2025.102534","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.datak.2025.102534","pdf_url":null,"source":{"id":"https://openalex.org/S136993123","display_name":"Data & Knowledge Engineering","issn_l":"0169-023X","issn":["0169-023X","1872-6933"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Data & Knowledge Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320336072","display_name":"Fakult\u00e4t f\u00fcr Informatik, Technische Universit\u00e4t M\u00fcnchen","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W2342249984","https://openalex.org/W2765320984","https://openalex.org/W2769041395","https://openalex.org/W2801952332","https://openalex.org/W2901499332","https://openalex.org/W2946475833","https://openalex.org/W2950680175","https://openalex.org/W3020745670","https://openalex.org/W3093927019","https://openalex.org/W3120891455","https://openalex.org/W3122436466","https://openalex.org/W3135028703","https://openalex.org/W3205626500","https://openalex.org/W4283731290","https://openalex.org/W4317352062","https://openalex.org/W4385416665","https://openalex.org/W4387144847","https://openalex.org/W4395462076","https://openalex.org/W4415230950"],"related_works":[],"abstract_inverted_index":{"In":[0,73],"general,":[1],"supervised":[2],"Machine":[3,30],"Learning":[4,31,106,113],"approaches":[5],"using":[6,98,104],"labeled":[7],"training":[8,39],"data":[9,22,40,55,64,69,83,89,202],"currently":[10],"promise":[11],"the":[12,61,68,87,102,127,138,145,152,161,165,174,196,206],"best":[13],"classification":[14,19,181],"results":[15],"with":[16,151,192],"respect":[17],"to":[18,59,134,172,211],"accuracy.":[20],"Therefore,":[21],"annotation":[23,65,70,84,159,203],"is":[24,42,53],"a":[25,38,78,109,120,193],"key":[26],"component":[27],"of":[28,81,91,111,124,140,157,164,176,198],"most":[29],"projects":[32,85,204],"implemented.":[33],"However,":[34],"creating":[35],"labels":[36,139,146],"for":[37,63,126,154,178],"set":[41,90],"often":[43,57],"an":[44],"elaborate":[45],"project":[46,143],"involving":[47],"arduous":[48],"and":[49,94,101,114,144,205,208],"repetitive":[50],"work,":[51],"which":[52],"why":[54],"scientists":[56],"try":[58],"minimize":[60],"effort":[62],"by":[66,148],"automating":[67],"process":[71],"itself.":[72],"this":[74],"paper,":[75],"we":[76,118,169],"present":[77],"case":[79],"study":[80],"two":[82],"on":[86],"same":[88],"support":[92,184],"tickets":[93],"compare":[95],"these:":[96],"one":[97],"human":[99,149,158,180],"annotators":[100],"other":[103],"algorithmic":[105],"Functions":[107],"in":[108,183,201],"combination":[110],"Active":[112],"Weak":[115],"Supervision.":[116],"Here,":[117],"achieved":[119],"weighted":[121],"confidence":[122],"score":[123],">94%":[125],"human-created":[128],"labels,":[129],"while":[130],"also":[131],"achieving":[132],"up":[133],"92%":[135],"agreement":[136],"between":[137],"our":[141],"automated":[142,166],"created":[147],"annotators,":[150],"need":[153],"only":[155],"10%":[156],"as":[160],"starting":[162],"input":[163],"approach.":[167],"Additionally,":[168],"were":[170],"able":[171],"reproduce":[173],"value":[175],"85%":[177],"initial":[179],"accuracy":[182],"ticket":[185,212],"distribution":[186],"from":[187],"previous":[188],"papers.":[189],"We":[190],"close":[191],"reflection":[194],"about":[195],"worth":[197],"business":[199],"understanding":[200],"problem":[207],"proposed":[209],"solutions":[210],"ambiguity.":[213]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-06-05T09:01:59.212387","created_date":"2025-11-23T00:00:00"}