{"id":"https://openalex.org/W4403674165","doi":"https://doi.org/10.15439/2024f3979","title":"Benchmarking OpenAI's APIs and Large Language Models for Repeatable, Efficient Question Answering Across Multiple Documents","display_name":"Benchmarking OpenAI's APIs and Large Language Models for Repeatable, Efficient Question Answering Across Multiple Documents","publication_year":2024,"publication_date":"2024-10-23","ids":{"openalex":"https://openalex.org/W4403674165","doi":"https://doi.org/10.15439/2024f3979"},"language":"en","primary_location":{"id":"doi:10.15439/2024f3979","is_oa":true,"landing_page_url":"https://doi.org/10.15439/2024f3979","pdf_url":"https://annals-csis.org/Volume_39/drp/pdf/3979.pdf","source":{"id":"https://openalex.org/S4220651875","display_name":"Annals of Computer Science and Information Systems","issn_l":"2300-5963","issn":["2300-5963"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":"https://openalex.org/P4310317484","host_organization_name":"Polskie Towarzystwo Informatyczne","host_organization_lineage":["https://openalex.org/P4310317484"],"host_organization_lineage_names":["Polskie Towarzystwo Informatyczne"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of Computer Science and Information Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://annals-csis.org/Volume_39/drp/pdf/3979.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114371497","display_name":"Elena Filipovska","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elena Filipovska","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114371498","display_name":"Ana Mladenovska","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ana Mladenovska","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087616882","display_name":"Merxhan Bajrami","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Merxhan Bajrami","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051889893","display_name":"Jovana Dobreva","orcid":"https://orcid.org/0000-0001-7382-2062"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jovana Dobreva","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Vellislava Hillman","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vellislava Hillman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030441606","display_name":"Petre Lameski","orcid":"https://orcid.org/0000-0002-5336-1796"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Petre Lameski","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5000303065","display_name":"Eftim Zdravevski","orcid":"https://orcid.org/0000-0001-7664-0168"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eftim Zdravevski","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7955,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.71731266,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"39","issue":null,"first_page":"107","last_page":"117"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9761000275611877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9473000168800354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.836348295211792},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7461315989494324},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5839574933052063},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5475699305534363},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5286576747894287},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4882626533508301},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.33250194787979126}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.836348295211792},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7461315989494324},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5839574933052063},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5475699305534363},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5286576747894287},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4882626533508301},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.33250194787979126},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.15439/2024f3979","is_oa":true,"landing_page_url":"https://doi.org/10.15439/2024f3979","pdf_url":"https://annals-csis.org/Volume_39/drp/pdf/3979.pdf","source":{"id":"https://openalex.org/S4220651875","display_name":"Annals of Computer Science and Information Systems","issn_l":"2300-5963","issn":["2300-5963"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":"https://openalex.org/P4310317484","host_organization_name":"Polskie Towarzystwo Informatyczne","host_organization_lineage":["https://openalex.org/P4310317484"],"host_organization_lineage_names":["Polskie Towarzystwo Informatyczne"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of Computer Science and Information Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:researchonline.lse.ac.uk:126674","is_oa":false,"landing_page_url":"https://doi.org/10.15439/2024F3979","pdf_url":null,"source":{"id":"https://openalex.org/S4306401593","display_name":"London School of Economics and Political Science Research Online (London School of Economics and Political Science)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I909854389","host_organization_name":"London School of Economics and Political Science","host_organization_lineage":["https://openalex.org/I909854389"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"},{"id":"pmh:oai:doaj.org/article:6b9cbf403e524b2ebb6bfb33340c504a","is_oa":false,"landing_page_url":"https://doaj.org/article/6b9cbf403e524b2ebb6bfb33340c504a","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Annals of computer science and information systems, Vol 39, Pp 107-117 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.15439/2024f3979","is_oa":true,"landing_page_url":"https://doi.org/10.15439/2024f3979","pdf_url":"https://annals-csis.org/Volume_39/drp/pdf/3979.pdf","source":{"id":"https://openalex.org/S4220651875","display_name":"Annals of Computer Science and Information Systems","issn_l":"2300-5963","issn":["2300-5963"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":"https://openalex.org/P4310317484","host_organization_name":"Polskie Towarzystwo Informatyczne","host_organization_lineage":["https://openalex.org/P4310317484"],"host_organization_lineage_names":["Polskie Towarzystwo Informatyczne"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annals of Computer Science and Information Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321691","display_name":"Universit\u00e4t Wien","ror":"https://ror.org/03prydq77"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403674165.pdf","grobid_xml":"https://content.openalex.org/works/W4403674165.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W4288267738","https://openalex.org/W2964413124","https://openalex.org/W4388937922","https://openalex.org/W3113264705"],"abstract_inverted_index":{"The":[0],"rapid":[1],"growth":[2],"of":[3,21,34,59,109,114,127,222,231],"document":[4,147,164,247],"volumes":[5],"and":[6,19,24,32,37,86,98,124,134,213,236],"complexity":[7],"in":[8,43,88,239,245],"various":[9],"domains":[10],"necessitates":[11],"advanced":[12],"automated":[13],"methods":[14],"to":[15,28,139,241],"enhance":[16],"the":[17,30,75,143,181,184,192,195,201,220,229],"efficiency":[18,31,235],"accuracy":[20],"information":[22],"extraction":[23],"analysis.This":[25],"paper":[26],"aims":[27,138],"evaluate":[29,105],"repeatability":[33],"OpenAI's":[35,76,153],"APIs":[36,87],"other":[38,182],"Large":[39],"Language":[40],"Models":[41],"(LLMs)":[42],"automating":[44],"question-answering":[45],"tasks":[46,73],"across":[47],"multiple":[48],"documents,":[49],"specifically":[50],"focusing":[51],"on":[52,69,219,227],"analyzing":[53],"Data":[54],"Privacy":[55],"Policy":[56],"(DPP)":[57],"documents":[58],"selected":[60],"EdTech":[61],"providers.We":[62],"test":[63],"how":[64],"well":[65],"these":[66],"models":[67,78,116,203],"perform":[68],"large-scale":[70],"text":[71],"processing":[72,217],"using":[74,152,167],"LLM":[77,115],"(GPT":[79],"3.5":[80],"Turbo,":[81],"GPT":[82,84],"4,":[83],"4o)":[85],"several":[89],"frameworks:":[90],"direct":[91],"API":[92,156],"calls":[93,157],"(i.e.,":[94],"one-shot":[95],"learning),":[96],"LangChain,":[97],"Retrieval":[99],"Augmented":[100],"Generation":[101],"(RAG)":[102],"systems.We":[103],"also":[104],"a":[106,125,159,168,174],"local":[107,169,185],"deployment":[108,186],"quantized":[110,202],"versions":[111],"(with":[112],"FAISS)":[113],"(Llama-2-13B-chat-GPTQ).Through":[117],"systematic":[118],"evaluation":[119],"against":[120],"predefined":[121],"use":[122,230],"cases":[123],"range":[126],"metrics,":[128],"including":[129],"response":[130],"format,":[131],"execution":[132],"time,":[133],"cost,":[135],"our":[136],"study":[137,224],"provide":[140],"insights":[141,226],"into":[142],"optimal":[144],"practices":[145],"for":[146,162,178,190,233],"analysis.Our":[148],"findings":[149,198],"demonstrate":[150],"that":[151,200],"LLMs":[154,232],"via":[155],"is":[158,172,187],"workable":[160],"workaround":[161],"accelerating":[163],"analysis":[165,248],"when":[166],"GPU-powered":[170],"infrastructure":[171],"not":[173,215],"viable":[175],"solution,":[176],"particularly":[177],"long":[179],"texts.On":[180],"hand,":[183],"quite":[188],"valuable":[189],"maintaining":[191],"data":[193,237],"within":[194],"private":[196],"infrastructure.Our":[197],"show":[199],"retain":[204],"substantial":[205],"relevance":[206],"even":[207],"with":[208],"fewer":[209],"parameters":[210],"than":[211],"ChatGPT":[212],"do":[214],"impose":[216],"restrictions":[218],"number":[221],"tokens.This":[223],"offers":[225],"maximizing":[228],"better":[234],"governance":[238],"addition":[240],"confirming":[242],"their":[243],"usefulness":[244],"improving":[246],"procedures.":[249]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}