{"id":"https://openalex.org/W4399317021","doi":"https://doi.org/10.48550/arxiv.2405.20389","title":"Designing an Evaluation Framework for Large Language Models in Astronomy Research","display_name":"Designing an Evaluation Framework for Large Language Models in Astronomy Research","publication_year":2024,"publication_date":"2024-05-30","ids":{"openalex":"https://openalex.org/W4399317021","doi":"https://doi.org/10.48550/arxiv.2405.20389"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.20389","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.20389","pdf_url":"https://arxiv.org/pdf/2405.20389","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.20389","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5072513422","display_name":"John F. Wu","orcid":"https://orcid.org/0000-0002-5077-881X"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Wu, John F.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099007984","display_name":"Alina Hyk","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hyk, Alina","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113238839","display_name":"Kiera McCormick","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"McCormick, Kiera","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021271786","display_name":"Christine Ye","orcid":"https://orcid.org/0000-0002-8559-0788"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ye, Christine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099007985","display_name":"Simone Astarita","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Astarita, Simone","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099007986","display_name":"Elina Baral","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baral, Elina","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099007987","display_name":"Jo Ciuca","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ciuca, Jo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040785005","display_name":"Jesse Cranney","orcid":"https://orcid.org/0000-0002-3015-9130"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cranney, Jesse","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113238840","display_name":"Anjalie Field","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Field, Anjalie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060659972","display_name":"Kartheik G. Iyer","orcid":"https://orcid.org/0000-0001-9298-3523"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Iyer, Kartheik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114172321","display_name":"Philipp Koehn","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koehn, Philipp","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133712280","display_name":"Jenn Kotler","orcid":"https://orcid.org/0000-0002-1294-7862"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kotler, Jenn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051876736","display_name":"Sandor Kruk","orcid":"https://orcid.org/0000-0001-8010-8879"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kruk, Sandor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070787181","display_name":"Michelle Ntampaka","orcid":"https://orcid.org/0000-0002-0144-387X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ntampaka, Michelle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112736126","display_name":"Charles O\u2019Neill","orcid":"https://orcid.org/0000-0003-2586-6874"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"O'Neill, Charles","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064943585","display_name":"J. E. G. Peek","orcid":"https://orcid.org/0000-0003-4797-7030"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peek, Joshua E. G.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034317478","display_name":"Sanjib Sharma","orcid":"https://orcid.org/0000-0002-0920-809X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sharma, Sanjib","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5049113660","display_name":"Mikaeel Yunus","orcid":"https://orcid.org/0000-0002-9851-2850"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yunus, Mikaeel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":18,"corresponding_author_ids":["https://openalex.org/A5072513422"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9117000102996826,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.9117000102996826,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/astronomy","display_name":"Astronomy","score":0.48660576343536377},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4255523085594177},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.24681547284126282}],"concepts":[{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.48660576343536377},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4255523085594177},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.24681547284126282}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.20389","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.20389","pdf_url":"https://arxiv.org/pdf/2405.20389","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.20389","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.20389","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.20389","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.20389","pdf_url":"https://arxiv.org/pdf/2405.20389","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4399317021.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4,80],"shifting":[5],"how":[6,16,23,57],"scientific":[7,24],"research":[8],"is":[9,12,34],"done.":[10],"It":[11],"imperative":[13],"to":[14,100,105],"understand":[15],"researchers":[17,59],"interact":[18,60],"with":[19,61,114],"these":[20,78],"models":[21],"and":[22,89,93,98,108,111],"sub-communities":[25],"like":[26],"astronomy":[27,58,83],"might":[28],"benefit":[29],"from":[30,72,85],"them.":[31],"However,":[32],"there":[33],"currently":[35],"no":[36],"standard":[37],"for":[38,52,129],"evaluating":[39],"the":[40,49,106,115],"use":[41],"of":[42,126],"LLMs":[43],"in":[44,82],"astronomy.":[45,130],"Therefore,":[46],"we":[47],"present":[48],"experimental":[50],"design":[51],"an":[53],"evaluation":[54],"study":[55],"on":[56],"LLMs.":[62],"We":[63,87],"deploy":[64],"a":[65],"Slack":[66],"chatbot":[67,94],"that":[68],"can":[69],"answer":[70],"queries":[71],"users":[73],"via":[74],"Retrieval-Augmented":[75],"Generation":[76],"(RAG);":[77],"responses":[79],"grounded":[81],"papers":[84],"arXiv.":[86],"record":[88],"anonymize":[90],"user":[91,96,103],"questions":[92],"answers,":[95],"upvotes":[97],"downvotes":[99],"LLM":[101,127],"responses,":[102],"feedback":[104],"LLM,":[107],"retrieved":[109],"documents":[110],"similarity":[112],"scores":[113],"query.":[116],"Our":[117],"data":[118],"collection":[119],"method":[120],"will":[121],"enable":[122],"future":[123],"dynamic":[124],"evaluations":[125],"tools":[128]},"counts_by_year":[],"updated_date":"2026-05-04T08:30:34.212998","created_date":"2024-06-04T00:00:00"}