{"id":"https://openalex.org/W4404088723","doi":"https://doi.org/10.48550/arxiv.2410.15512","title":"Reverse Question Answering: Can an LLM Write a Question so Hard (or Bad) that it Can't Answer?","display_name":"Reverse Question Answering: Can an LLM Write a Question so Hard (or Bad) that it Can't Answer?","publication_year":2024,"publication_date":"2024-10-20","ids":{"openalex":"https://openalex.org/W4404088723","doi":"https://doi.org/10.48550/arxiv.2410.15512"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.15512","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.15512","pdf_url":"https://arxiv.org/pdf/2410.15512","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.15512","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091963084","display_name":"Nishant Balepur","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Balepur, Nishant","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113447185","display_name":"Feng Gu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gu, Feng","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021017923","display_name":"Abhilasha Ravichander","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ravichander, Abhilasha","raw_affiliation_strings":["University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057958949","display_name":"Shi Feng","orcid":"https://orcid.org/0000-0002-2846-7652","observed_orcids":["https://orcid.org/0000-0002-2846-7652"]},"institutions":[{"id":"https://openalex.org/I193531525","display_name":"George Washington University","ror":"https://ror.org/00y4zzh67","country_code":"US","type":"education","lineage":["https://openalex.org/I193531525"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Feng, Shi","raw_affiliation_strings":["George Washington University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"George Washington University","institution_ids":["https://openalex.org/I193531525"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081307846","display_name":"Jordan Lee Boyd-Graber","orcid":"https://orcid.org/0000-0002-7770-4431","observed_orcids":["https://orcid.org/0000-0002-7770-4431"]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Boyd-Graber, Jordan","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082447472","display_name":"Rachel Rudinger","orcid":"https://orcid.org/0000-0002-5506-4701","observed_orcids":["https://orcid.org/0000-0002-5506-4701"]},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rudinger, Rachel","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9692000150680542,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9692000150680542,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.017400000244379044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.008500000461935997,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/question-generation","display_name":"question generation","score":0.6669999957084656},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8960000276565552},{"id":"https://openalex.org/keywords/question-difficulty","display_name":"question difficulty","score":0.6809999942779541},{"id":"https://openalex.org/keywords/reasoning-consistency","display_name":"reasoning consistency","score":0.6629999876022339},{"id":"https://openalex.org/keywords/benchmark-design","display_name":"benchmark design","score":0.7490000128746033},{"id":"https://openalex.org/keywords/trivia-questions","display_name":"trivia questions","score":0.4909999966621399}],"concepts":[{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.792709231376648},{"id":"https://openalex.org/C3019144022","wikidata":"https://www.wikidata.org/wiki/Q4124998","display_name":"Questions and answers","level":2,"score":0.4754347801208496},{"id":"https://openalex.org/C193035329","wikidata":"https://www.wikidata.org/wiki/Q17007046","display_name":"Closed-ended question","level":2,"score":0.4626994729042053},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4234657287597656},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.36754825711250305},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.32252079248428345},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.28214675188064575}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.15512","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.15512","pdf_url":"https://arxiv.org/pdf/2410.15512","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.15512","doi":"https://doi.org/10.48550/arxiv.2410.15512","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.15512","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.15512","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.15512","pdf_url":"https://arxiv.org/pdf/2410.15512","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G5052875863","display_name":"Collaborative Research: RI: Medium: Hard Data to the Model: Personalized, Diverse Preferences for Language Models","funder_award_id":"2403436","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6317553775","display_name":"Graduate Research Fellowship Program (GRFP)","funder_award_id":"2236417","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8830764304","display_name":"CAREER: Robust, Fair, and Culturally Aware Commonsense Reasoning in Natural Language","funder_award_id":"2339746","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8979874499","display_name":"Scaling Laws of Deep Learning","funder_award_id":"2134012","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404088723.pdf","grobid_xml":"https://content.openalex.org/works/W4404088723.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2118632115","https://openalex.org/W2555213299","https://openalex.org/W2558460724","https://openalex.org/W2576107865","https://openalex.org/W3215363805","https://openalex.org/W1846393350","https://openalex.org/W3154330131","https://openalex.org/W4307481286","https://openalex.org/W1990306092","https://openalex.org/W3107555006"],"abstract_inverted_index":{"Question":[0],"answering":[1,17],"(QA),":[2],"giving":[3],"correct":[4],"answers":[5],"to":[6,128,141],"questions,":[7],"is":[8],"a":[9,24],"popular":[10],"task,":[11],"but":[12,36,76],"we":[13,37,144],"test":[14,38],"reverse":[15],"question":[16,25,112,135],"(RQA):":[18],"for":[19,73,82,147],"an":[20],"input":[21],"answer,":[22],"give":[23],"with":[26,59,111,117],"that":[27,139],"answer.":[28],"Past":[29],"work":[30],"tests":[31],"QA":[32,56],"and":[33,47,57,114,124,136],"RQA":[34,58,72,81,94,99,108,142],"separately,":[35],"them":[39],"jointly,":[40],"comparing":[41],"their":[42,89],"difficulty,":[43],"aiding":[44],"benchmark":[45],"design,":[46],"checking":[48],"reasoning":[49],"consistency.":[50],"We":[51],"run":[52],"16":[53],"LLMs":[54,66,86,126],"on":[55],"trivia":[60],"questions/answers,":[61],"revealing:":[62],"1)":[63],"Versus":[64],"QA,":[65,97],"are":[67,101],"much":[68],"less":[69],"accurate":[70,79],"in":[71,80,96,120],"numerical":[74],"answers,":[75],"slightly":[77],"more":[78],"textual":[83],"answers;":[84],"2)":[85],"often":[87],"answer":[88,118,137],"own":[90],"invalid":[91],"questions":[92],"from":[93,103],"accurately":[95],"so":[98],"errors":[100,109],"not":[102],"knowledge":[104],"gaps":[105],"alone;":[106],"3)":[107],"correlate":[110,116],"difficulty":[113],"inversely":[115],"frequencies":[119],"the":[121],"Dolma":[122],"corpus;":[123],"4)":[125],"struggle":[127],"provide":[129],"valid":[130],"multi-hop":[131],"questions.":[132],"By":[133],"finding":[134],"types":[138],"lead":[140],"errors,":[143],"suggest":[145],"improvements":[146],"LLM":[148],"reasoning.":[149]},"counts_by_year":[],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2025-10-10T00:00:00"}