{"id":"https://openalex.org/W3090476742","doi":"https://doi.org/10.1145/3394171.3413749","title":"Describing Subjective Experiment Consistency by p-Value P--P Plot","display_name":"Describing Subjective Experiment Consistency by p-Value P--P Plot","publication_year":2020,"publication_date":"2020-10-12","ids":{"openalex":"https://openalex.org/W3090476742","doi":"https://doi.org/10.1145/3394171.3413749","mag":"3090476742"},"language":"en","primary_location":{"id":"doi:10.1145/3394171.3413749","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394171.3413749","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2009.13372","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000953614","display_name":"Jakub Nawa\u0142a","orcid":"https://orcid.org/0000-0001-8367-4577"},"institutions":[{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Jakub Nawala","raw_affiliation_strings":["AGH University of Science and Technology, Krak\u00f3w, Poland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AGH University of Science and Technology, Krak\u00f3w, Poland","institution_ids":["https://openalex.org/I686019"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064855801","display_name":"Lucjan Janowski","orcid":"https://orcid.org/0000-0002-3151-2944"},"institutions":[{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Lucjan Janowski","raw_affiliation_strings":["AGH University of Science and Technology, Krak\u00f3w, Poland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AGH University of Science and Technology, Krak\u00f3w, Poland","institution_ids":["https://openalex.org/I686019"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062706974","display_name":"Bogdan \u0106miel","orcid":"https://orcid.org/0000-0002-2284-6360"},"institutions":[{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Bogdan Cmiel","raw_affiliation_strings":["AGH University of Science and Technology, Krak\u00f3w, Poland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AGH University of Science and Technology, Krak\u00f3w, Poland","institution_ids":["https://openalex.org/I686019"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047003405","display_name":"Krzysztof Rusek","orcid":"https://orcid.org/0000-0003-4336-7841"},"institutions":[{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Krzysztof Rusek","raw_affiliation_strings":["AGH University of Science and Technology, Krak\u00f3w, Poland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AGH University of Science and Technology, Krak\u00f3w, Poland","institution_ids":["https://openalex.org/I686019"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I686019"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"852","last_page":"861"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11871","display_name":"Advanced Statistical Methods and Models","score":0.9218999743461609,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.699690043926239},{"id":"https://openalex.org/keywords/trustworthiness","display_name":"Trustworthiness","score":0.6547285318374634},{"id":"https://openalex.org/keywords/plot","display_name":"Plot (graphics)","score":0.6019281148910522},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5974411368370056},{"id":"https://openalex.org/keywords/goodness-of-fit","display_name":"Goodness of fit","score":0.5629565715789795},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5584871172904968},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4913143217563629},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.454105943441391},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3926513195037842},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38610637187957764},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.36576879024505615},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.31802934408187866},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.26176249980926514}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.699690043926239},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.6547285318374634},{"id":"https://openalex.org/C167651023","wikidata":"https://www.wikidata.org/wiki/Q1474611","display_name":"Plot (graphics)","level":2,"score":0.6019281148910522},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5974411368370056},{"id":"https://openalex.org/C132480984","wikidata":"https://www.wikidata.org/wiki/Q2034239","display_name":"Goodness of fit","level":2,"score":0.5629565715789795},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5584871172904968},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4913143217563629},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.454105943441391},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3926513195037842},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38610637187957764},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36576879024505615},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.31802934408187866},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.26176249980926514},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3394171.3413749","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3394171.3413749","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 28th ACM International Conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2009.13372","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2009.13372","pdf_url":"https://arxiv.org/pdf/2009.13372","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2009.13372","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2009.13372","pdf_url":"https://arxiv.org/pdf/2009.13372","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W127357287","https://openalex.org/W149213537","https://openalex.org/W192654931","https://openalex.org/W272187772","https://openalex.org/W1966159961","https://openalex.org/W1966700952","https://openalex.org/W1981546016","https://openalex.org/W1982732426","https://openalex.org/W1995945562","https://openalex.org/W2011087550","https://openalex.org/W2013861934","https://openalex.org/W2022359415","https://openalex.org/W2026874689","https://openalex.org/W2047480396","https://openalex.org/W2057758736","https://openalex.org/W2076983043","https://openalex.org/W2145844835","https://openalex.org/W2158741413","https://openalex.org/W2162980932","https://openalex.org/W2177315755","https://openalex.org/W2295519251","https://openalex.org/W2333897677","https://openalex.org/W2474880636","https://openalex.org/W2500862931","https://openalex.org/W2561313130","https://openalex.org/W2562524806","https://openalex.org/W2573292679","https://openalex.org/W2579555219","https://openalex.org/W2725569041","https://openalex.org/W2785907871","https://openalex.org/W2806526132","https://openalex.org/W2891989842","https://openalex.org/W2892866971","https://openalex.org/W2894154471","https://openalex.org/W2896066866","https://openalex.org/W2920978843","https://openalex.org/W2940611027","https://openalex.org/W2949874897","https://openalex.org/W2956133448","https://openalex.org/W2963398794","https://openalex.org/W2963562284","https://openalex.org/W2972990032","https://openalex.org/W3049619335","https://openalex.org/W3099045918","https://openalex.org/W3103446595","https://openalex.org/W3106889297","https://openalex.org/W3148106702","https://openalex.org/W4235243453","https://openalex.org/W4242499965","https://openalex.org/W4248173958","https://openalex.org/W4302583945","https://openalex.org/W4302768905","https://openalex.org/W6655209129"],"related_works":["https://openalex.org/W2366107444","https://openalex.org/W4388145910","https://openalex.org/W2381570729","https://openalex.org/W1976205134","https://openalex.org/W4248336175","https://openalex.org/W2031260042","https://openalex.org/W2391445434","https://openalex.org/W3009369890","https://openalex.org/W1541538682","https://openalex.org/W1989708511"],"abstract_inverted_index":{"There":[0],"are":[1,182],"phenomena":[2],"that":[3,155,174],"cannot":[4],"be":[5,68],"measured":[6],"without":[7],"subjective":[8,11,38,65,99,125,166,206],"testing.":[9],"However,":[10],"testing":[12],"is":[13,46,70,121],"a":[14,32,95,128,144],"complex":[15],"issue":[16],"with":[17,143,193],"many":[18],"influencing":[19],"factors.":[20],"These":[21],"interplay":[22],"to":[23,34,50,53,72,97,150],"yield":[24],"either":[25,41,106],"precise":[26,184],"or":[27,43,108],"incorrect":[28],"results.":[29],"Researchers":[30],"require":[31],"tool":[33,96,112,160],"classify":[35,98],"results":[36],"of":[37,90,147,201],"experiment":[39,100,198],"as":[40,58,105,127],"consistent":[42,107],"inconsistent.":[44,109],"This":[45],"necessary":[47],"in":[48,141,191],"order":[49],"decide":[51],"whether":[52],"treat":[54],"the":[55,87,111,133],"gathered":[56],"scores":[57,66,83,126],"quality":[59,89,205],"ground":[60],"truth":[61],"data.":[62,167],"Knowing":[63],"if":[64],"can":[67],"trusted":[69],"key":[71],"drawing":[73],"valid":[74],"conclusions":[75,175],"and":[76,178,185],"building":[77],"functional":[78],"tools":[79,179],"based":[80,122],"on":[81,123,197],"those":[82],"(e.g.,":[84],"algorithms":[85],"assessing":[86],"perceived":[88],"multimedia":[91,204],"materials).":[92],"We":[93],"provide":[94],"(and":[101],"all":[102],"its":[103],"results)":[104],"Additionally,":[110],"identifies":[113],"stimuli":[114],"having":[115],"irregular":[116],"score":[117],"distribution.":[118],"The":[119,139,159,187],"approach":[120,189],"treating":[124],"random":[129],"variable":[130],"coming":[131],"from":[132,163],"discrete":[134],"Generalized":[135],"Score":[136],"Distribution":[137],"(GSD).":[138],"GSD,":[140],"combination":[142],"bootstrapped":[145],"G-test":[146],"goodness-of-fit,":[148],"allows":[149],"construct":[151],"p-value":[152],"P--P":[153],"plot":[154],"visualizes":[156],"experiment's":[157],"consistency.":[158],"safeguards":[161],"researchers":[162],"using":[164],"inconsistent":[165],"In":[168],"this":[169],"way,":[170],"it":[171],"makes":[172],"sure":[173],"they":[176,180],"draw":[177],"build":[181],"more":[183],"trustworthy.":[186],"proposed":[188],"works":[190],"line":[192],"expectations":[194],"drawn":[195],"solely":[196],"design":[199],"descriptions":[200],"21":[202],"real-life":[203],"experiments.":[207]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":5}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2020-10-08T00:00:00"}