{"id":"https://openalex.org/W4414230825","doi":"https://doi.org/10.1109/sera65747.2025.11154541","title":"SYNC: SYnergistic aNnotation Collaboration between Humans and LLMs for Enhanced Model Training","display_name":"SYNC: SYnergistic aNnotation Collaboration between Humans and LLMs for Enhanced Model Training","publication_year":2025,"publication_date":"2025-05-29","ids":{"openalex":"https://openalex.org/W4414230825","doi":"https://doi.org/10.1109/sera65747.2025.11154541"},"language":"en","primary_location":{"id":"doi:10.1109/sera65747.2025.11154541","is_oa":false,"landing_page_url":"https://doi.org/10.1109/sera65747.2025.11154541","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACIS 23rd International Conference on Software Engineering Research, Management and Applications (SERA)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068228116","display_name":"Tung B. K. Le","orcid":"https://orcid.org/0000-0003-4764-8851"},"institutions":[{"id":"https://openalex.org/I122266389","display_name":"University of Nebraska at Omaha","ror":"https://ror.org/04yrkc140","country_code":"US","type":"education","lineage":["https://openalex.org/I122266389"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Tammy Le","raw_affiliation_strings":["University of Nebraska at Omaha,Computer Science,Nebraska,USA"],"affiliations":[{"raw_affiliation_string":"University of Nebraska at Omaha,Computer Science,Nebraska,USA","institution_ids":["https://openalex.org/I122266389"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100776609","display_name":"William J. Taylor","orcid":"https://orcid.org/0000-0001-6075-8479"},"institutions":[{"id":"https://openalex.org/I122266389","display_name":"University of Nebraska at Omaha","ror":"https://ror.org/04yrkc140","country_code":"US","type":"education","lineage":["https://openalex.org/I122266389"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Will Taylor","raw_affiliation_strings":["University of Nebraska at Omaha,Computer Science,Nebraska,USA"],"affiliations":[{"raw_affiliation_string":"University of Nebraska at Omaha,Computer Science,Nebraska,USA","institution_ids":["https://openalex.org/I122266389"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117495635","display_name":"Shradha Maharjan","orcid":null},"institutions":[{"id":"https://openalex.org/I122266389","display_name":"University of Nebraska at Omaha","ror":"https://ror.org/04yrkc140","country_code":"US","type":"education","lineage":["https://openalex.org/I122266389"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shradha Maharjan","raw_affiliation_strings":["University of Nebraska at Omaha,Computer Science,Nebraska,USA"],"affiliations":[{"raw_affiliation_string":"University of Nebraska at Omaha,Computer Science,Nebraska,USA","institution_ids":["https://openalex.org/I122266389"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035687843","display_name":"Meng Xia","orcid":"https://orcid.org/0000-0002-2676-9032"},"institutions":[{"id":"https://openalex.org/I122266389","display_name":"University of Nebraska at Omaha","ror":"https://ror.org/04yrkc140","country_code":"US","type":"education","lineage":["https://openalex.org/I122266389"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Meng Xia","raw_affiliation_strings":["University of Nebraska at Omaha,Computer Science,Nebraska,USA"],"affiliations":[{"raw_affiliation_string":"University of Nebraska at Omaha,Computer Science,Nebraska,USA","institution_ids":["https://openalex.org/I122266389"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035020782","display_name":"Myoungkyu Song","orcid":"https://orcid.org/0000-0003-4477-8933"},"institutions":[{"id":"https://openalex.org/I122266389","display_name":"University of Nebraska at Omaha","ror":"https://ror.org/04yrkc140","country_code":"US","type":"education","lineage":["https://openalex.org/I122266389"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Myoungkyu Song","raw_affiliation_strings":["University of Nebraska at Omaha,Computer Science,Nebraska,USA"],"affiliations":[{"raw_affiliation_string":"University of Nebraska at Omaha,Computer Science,Nebraska,USA","institution_ids":["https://openalex.org/I122266389"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5068228116"],"corresponding_institution_ids":["https://openalex.org/I122266389"],"apc_list":null,"apc_paid":null,"fwci":1.3682,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.84945033,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"158","last_page":"162"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14470","display_name":"Advanced Data Processing Techniques","score":0.8560000061988831,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14470","display_name":"Advanced Data Processing Techniques","score":0.8560000061988831,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8210999965667725,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.8012999892234802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.886900007724762},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6349999904632568},{"id":"https://openalex.org/keywords/sync","display_name":"sync","score":0.4205000102519989},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.39660000801086426},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.36329999566078186},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.34950000047683716},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.34549999237060547},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.3084999918937683}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.886900007724762},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7731000185012817},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6349999904632568},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4578999876976013},{"id":"https://openalex.org/C3913047","wikidata":"https://www.wikidata.org/wiki/Q1956265","display_name":"sync","level":3,"score":0.4205000102519989},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.39660000801086426},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37380000948905945},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3644999861717224},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.36329999566078186},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.36039999127388},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3544999957084656},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.34950000047683716},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.34549999237060547},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3084999918937683},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.301800012588501},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.29440000653266907},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.29409998655319214},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.28690001368522644},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.27570000290870667},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.27059999108314514},{"id":"https://openalex.org/C187191949","wikidata":"https://www.wikidata.org/wiki/Q1138496","display_name":"Profiling (computer programming)","level":2,"score":0.26499998569488525},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C91632574","wikidata":"https://www.wikidata.org/wiki/Q15088675","display_name":"Data curation","level":2,"score":0.2540000081062317}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/sera65747.2025.11154541","is_oa":false,"landing_page_url":"https://doi.org/10.1109/sera65747.2025.11154541","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACIS 23rd International Conference on Software Engineering Research, Management and Applications (SERA)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W2970641574","https://openalex.org/W3098605233","https://openalex.org/W3108032709","https://openalex.org/W3121707215","https://openalex.org/W3161997752","https://openalex.org/W4221166942","https://openalex.org/W4284697810","https://openalex.org/W4392414327","https://openalex.org/W4401042461"],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1,14],"models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"impressive":[6],"performance":[7],"across":[8],"a":[9,67,143],"wide":[10],"range":[11],"of":[12,47,112],"natural":[13],"processing":[15],"tasks,":[16],"highlighting":[17],"their":[18,100],"potential":[19,127],"as":[20,193],"effective":[21],"data":[22,74],"annotators.":[23],"While":[24],"LLM-generated":[25],"annotations":[26],"tend":[27],"to":[28,43,52,72,85,98,125,175,182],"be":[29],"costeffective,":[30],"they":[31],"are":[32],"often":[33],"error-prone":[34],"and":[35,50,56,87,96,109,129,163],"may":[36],"inadvertently":[37],"introduce":[38],"bias.":[39],"It":[40],"is":[41,173],"advantageous":[42],"harness":[44],"the":[45,59,106,164],"strengths":[46],"both":[48],"LLMs":[49],"humans":[51],"ensure":[53],"higher":[54],"accuracy":[55],"reliability":[57],"in":[58,116,196],"annotation":[60,75,184],"process.":[61],"In":[62],"this":[63],"paper,":[64],"we":[65,92,140],"present":[66],"multi-step,":[68],"human-LLM":[69,138],"collaborative":[70],"approach":[71,172],"optimize":[73],"for":[76,104,167,187],"Stack":[77,113,197],"Overflow":[78,198],"datasets.":[79,199],"We":[80],"begin":[81],"by":[82],"applying":[83],"TF-IDF":[84],"rank":[86],"prioritize":[88],"relevant":[89],"elements.":[90],"Next,":[91],"utilize":[93],"NLP":[94],"Transformer":[95],"UniXcoder":[97],"leverage":[99],"deep":[101],"contextual":[102],"understanding":[103],"handling":[105],"code-related":[107],"queries":[108],"discussions":[110],"typical":[111],"Overflow,":[114],"resulting":[115],"more":[117],"consistent":[118],"automated":[119,177],"labels.":[120],"Finally,":[121],"human":[122,161,180],"annotators":[123,162],"re-annotate":[124],"correct":[126],"errors":[128],"mitigate":[130],"bias":[131],"introduced":[132],"during":[133],"earlier":[134],"stages.":[135],"To":[136],"support":[137],"collaboration,":[139],"developed":[141],"SYNC":[142],"research":[144],"prototype":[145],"that":[146],"implements":[147],"SYnergistic":[148],"aNnotation":[149],"Collaboration":[150],"through":[151],"an":[152],"intuitive":[153],"graphical":[154],"user":[155],"interface,":[156],"enabling":[157],"real-time":[158],"interaction":[159],"between":[160],"LLM,":[165],"allowing":[166],"iterative":[168],"refinements.":[169],"Overall,":[170],"our":[171],"designed":[174],"integrate":[176],"efficiency":[178],"with":[179],"oversight":[181],"improve":[183],"outcomes,":[185],"particularly":[186],"complex":[188],"or":[189],"domain-specific":[190],"tasks":[191],"such":[192],"those":[194],"found":[195]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-03-07T16:01:11.037858","created_date":"2025-10-10T00:00:00"}