{"id":"https://openalex.org/W7154846867","doi":"https://doi.org/10.1145/3805712.3808542","title":"Evaluation of Agents under Simulated AI Marketplace Dynamics","display_name":"Evaluation of Agents under Simulated AI Marketplace Dynamics","publication_year":2026,"publication_date":"2026-07-10","ids":{"openalex":"https://openalex.org/W7154846867","doi":"https://doi.org/10.1145/3805712.3808542"},"language":null,"primary_location":{"id":"doi:10.1145/3805712.3808542","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808542","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3805712.3808542","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005474453","display_name":"To Eun Kim","orcid":"https://orcid.org/0000-0002-2807-1623"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"To Eun Kim","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-2807-1623","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082594835","display_name":"Alireza Salemi","orcid":"https://orcid.org/0009-0006-1937-2615"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alireza Salemi","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, MA, USA"],"raw_orcid":"https://orcid.org/0009-0006-1937-2615","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, MA, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101457713","display_name":"Hamed Zamani","orcid":"https://orcid.org/0000-0002-0800-3340"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hamed Zamani","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, MA, USA"],"raw_orcid":"https://orcid.org/0000-0002-0800-3340","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, MA, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5133912723","display_name":"Fernando Diaz","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fernando Diaz","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":"https://orcid.org/0000-0003-2345-1288","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2672","last_page":"2683"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.1777999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.1777999997138977,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.04859999939799309,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.043800000101327896,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.607699990272522},{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.6039000153541565},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5496000051498413},{"id":"https://openalex.org/keywords/competition","display_name":"Competition (biology)","score":0.5270000100135803},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.43779999017715454},{"id":"https://openalex.org/keywords/competitive-advantage","display_name":"Competitive advantage","score":0.3984000086784363},{"id":"https://openalex.org/keywords/system-dynamics","display_name":"System dynamics","score":0.36559998989105225}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7246999740600586},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.607699990272522},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.6039000153541565},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5496000051498413},{"id":"https://openalex.org/C91306197","wikidata":"https://www.wikidata.org/wiki/Q45767","display_name":"Competition (biology)","level":2,"score":0.5270000100135803},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.43779999017715454},{"id":"https://openalex.org/C58546491","wikidata":"https://www.wikidata.org/wiki/Q1150207","display_name":"Competitive advantage","level":2,"score":0.3984000086784363},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.39239999651908875},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.36910000443458557},{"id":"https://openalex.org/C77405623","wikidata":"https://www.wikidata.org/wiki/Q598451","display_name":"System dynamics","level":2,"score":0.36559998989105225},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.357699990272522},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.3409999907016754},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.30480000376701355},{"id":"https://openalex.org/C91262260","wikidata":"https://www.wikidata.org/wiki/Q528074","display_name":"End user","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.29440000653266907},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2897999882698059},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2651999890804291},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2646999955177307},{"id":"https://openalex.org/C2985380958","wikidata":"https://www.wikidata.org/wiki/Q319676","display_name":"Market competition","level":2,"score":0.25870001316070557}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3805712.3808542","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808542","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2604.14256","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2604.14256","pdf_url":"https://arxiv.org/pdf/2604.14256","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3805712.3808542","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3805712.3808542","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 49th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Life in Land","score":0.4108353853225708,"id":"https://metadata.un.org/sdg/15"}],"awards":[{"id":"https://openalex.org/G266314255","display_name":"Collaborative Research: III: Medium: Retrieval-Enhanced Machine Learning Through an Information Retrieval Lens","funder_award_id":"2402874","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6256039072","display_name":"Collaborative Research: III: Medium: Retrieval-Enhanced Machine Learning Through an Information Retrieval Lens","funder_award_id":"2402873","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Modern":[0],"information":[1,108],"access":[2,24,109],"ecosystems":[3],"consist":[4],"of":[5,7],"mixtures":[6],"systems,":[8],"such":[9,38,92,135],"as":[10,93,111,136],"retrieval":[11],"systems":[12,33,75,110],"and":[13,17,28,58,88,96,121,124,132,138,143,154,162,169],"large":[14],"language":[15],"models,":[16,26],"increasingly":[18],"rely":[19],"on":[20,67],"marketplaces":[21],"to":[22,25,35,84],"mediate":[23],"tools,":[27],"data,":[29],"making":[30],"competition":[31],"between":[32],"inherent":[34],"deployment.":[36],"In":[37],"settings,":[39],"outcomes":[40],"are":[41],"shaped":[42],"not":[43],"only":[44],"by":[45,50,160],"benchmark":[46],"quality":[47],"but":[48],"also":[49],"competitive":[51,90,115],"pressure,":[52],"including":[53],"user":[54,123],"switching,":[55],"routing":[56],"decisions,":[57],"operational":[59],"constraints.":[60],"Yet":[61],"evaluation":[62,131,172],"is":[63],"still":[64],"largely":[65],"conducted":[66],"static":[68],"benchmarks":[69],"with":[70],"accuracy-focused":[71],"measures":[72],"that":[73,106,141],"assume":[74],"operate":[76],"in":[77,113,171],"isolation.":[78],"This":[79],"mismatch":[80],"makes":[81],"it":[82],"difficult":[83],"predict":[85],"post-deployment":[86],"success":[87],"obscures":[89],"effects":[91],"early-adoption":[94],"advantages":[95],"market":[97,139],"dominance.":[98],"We":[99,150],"introduce":[100],"Marketplace":[101],"Evaluation,":[102],"a":[103,114,156],"simulation-based":[104],"paradigm":[105],"evaluates":[107],"participants":[112],"marketplace.":[116],"By":[117],"simulating":[118],"repeated":[119],"interactions":[120],"evolving":[122],"agent":[125],"preferences,":[126],"the":[127,152],"framework":[128,153],"enables":[129],"longitudinal":[130],"marketplace-level":[133],"metrics,":[134,167],"retention":[137],"share,":[140],"complement":[142],"can":[144],"extend":[145],"beyond":[146],"traditional":[147],"accuracy-based":[148],"metrics.":[149],"formalize":[151],"outline":[155],"research":[157],"agenda,":[158],"motivated":[159],"business":[161],"economics,":[163],"around":[164],"marketplace":[165],"simulation,":[166],"optimization,":[168],"adoption":[170],"campaigns":[173],"like":[174],"TREC.":[175]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-19T00:00:00"}