{"id":"https://openalex.org/W4417529522","doi":"https://doi.org/10.1145/3786149.3788306","title":"OLAF: Towards Robust LLM-Based Annotation Framework in Empirical Software Engineering","display_name":"OLAF: Towards Robust LLM-Based Annotation Framework in Empirical Software Engineering","publication_year":2026,"publication_date":"2026-04-12","ids":{"openalex":"https://openalex.org/W4417529522","doi":"https://doi.org/10.1145/3786149.3788306"},"language":"en","primary_location":{"id":"doi:10.1145/3786149.3788306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786149.3788306","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 IEEE/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3786149.3788306","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088063511","display_name":"Mia Mohammad Imran","orcid":"https://orcid.org/0000-0003-2477-9971"},"institutions":[{"id":"https://openalex.org/I20382870","display_name":"Missouri University of Science and Technology","ror":"https://ror.org/00scwqd12","country_code":"US","type":"education","lineage":["https://openalex.org/I20382870"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mia Mohammad Imran","raw_affiliation_strings":["Computer Science, Missouri University of Science and Technology, Rolla, MO, USA"],"raw_orcid":"https://orcid.org/0000-0003-2477-9971","affiliations":[{"raw_affiliation_string":"Computer Science, Missouri University of Science and Technology, Rolla, MO, USA","institution_ids":["https://openalex.org/I20382870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050415345","display_name":"Tarannum Shaila Zaman","orcid":"https://orcid.org/0000-0002-8634-524X"},"institutions":[{"id":"https://openalex.org/I79272384","display_name":"University of Maryland, Baltimore County","ror":"https://ror.org/02qskvh78","country_code":"US","type":"education","lineage":["https://openalex.org/I79272384"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tarannum Shaila Zaman","raw_affiliation_strings":["Information Systems, University of Maryland Baltimore County, Baltimore, MF, USA"],"raw_orcid":"https://orcid.org/0000-0002-8634-524X","affiliations":[{"raw_affiliation_string":"Information Systems, University of Maryland Baltimore County, Baltimore, MF, USA","institution_ids":["https://openalex.org/I79272384"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.02097169,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"33","last_page":"36"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.31119999289512634,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.31119999289512634,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.11800000071525574,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.1046999990940094,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.8431000113487244},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5047000050544739},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4546999931335449},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4309000074863434},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.42010000348091125},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.41909998655319214},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.41620001196861267},{"id":"https://openalex.org/keywords/position-paper","display_name":"Position paper","score":0.36000001430511475}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8431000113487244},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7249000072479248},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5949000120162964},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5047000050544739},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4546999931335449},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4309000074863434},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.42010000348091125},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.41909998655319214},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.41620001196861267},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4000000059604645},{"id":"https://openalex.org/C78780964","wikidata":"https://www.wikidata.org/wiki/Q7233193","display_name":"Position paper","level":2,"score":0.36000001430511475},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.35249999165534973},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.3517000079154968},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3449000120162964},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.314300000667572},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.30889999866485596},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30149999260902405},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.27619999647140503},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.27469998598098755},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2727000117301941},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.27250000834465027},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.258899986743927}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3786149.3788306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786149.3788306","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 IEEE/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2512.15979","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2512.15979","pdf_url":"https://arxiv.org/pdf/2512.15979","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:mdsoar.org:11603/41542","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arXiv.2512.15979","pdf_url":null,"source":{"id":"https://openalex.org/S4306402556","display_name":"Maryland Shared Open Access Repository (USMAI Consortium)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"},{"id":"doi:10.48550/arxiv.2512.15979","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2512.15979","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3786149.3788306","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786149.3788306","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 IEEE/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4],"increasingly":[5],"used":[6],"in":[7,116],"empirical":[8,107],"software":[9,117],"engineering":[10,118],"(ESE)":[11],"to":[12,101],"automate":[13],"or":[14],"assist":[15],"annotation":[16,57,115],"tasks":[17],"such":[18,32],"as":[19,61],"labeling":[20],"commits,":[21],"issues,":[22],"and":[23,29,45,47,96,105,112],"qualitative":[24],"artifacts.":[25],"Yet":[26],"the":[27,77],"reliability":[28],"reproducibility":[30],"of":[31],"annotations":[33],"remain":[34],"underexplored.":[35],"Existing":[36],"studies":[37],"often":[38],"lack":[39],"standardized":[40],"measures":[41],"for":[42,79],"reliability,":[43],"calibration,":[44,92],"drift,":[46,93],"frequently":[48],"omit":[49],"essential":[50],"configuration":[51],"details.":[52],"We":[53],"argue":[54],"that":[55,87],"LLM-based":[56,80,114],"should":[58],"be":[59],"treated":[60],"a":[62,67,84],"measurement":[63],"process":[64],"rather":[65],"than":[66],"purely":[68],"automated":[69],"activity.":[70],"In":[71],"this":[72],"position":[73],"paper,":[74],"we":[75],"outline":[76],"\\textbf{Operationalization":[78],"Annotation":[81],"Framework":[82],"(OLAF)},":[83],"conceptual":[85],"framework":[86],"organizes":[88],"key":[89],"constructs:":[90],"\\textit{reliability,":[91],"consensus,":[94],"aggregation},":[95],"\\textit{transparency}.":[97],"The":[98],"paper":[99],"aims":[100],"motivate":[102],"methodological":[103],"discussion":[104],"future":[106],"work":[108],"toward":[109],"more":[110],"transparent":[111],"reproducible":[113],"research.":[119]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-12-21T00:00:00"}