{"id":"https://openalex.org/W2104508137","doi":"https://doi.org/10.1017/s1351324909005051","title":"Formal and functional assessment of the pyramid method for summary content evaluation","display_name":"Formal and functional assessment of the pyramid method for summary content evaluation","publication_year":2009,"publication_date":"2009-04-06","ids":{"openalex":"https://openalex.org/W2104508137","doi":"https://doi.org/10.1017/s1351324909005051","mag":"2104508137"},"language":"en","primary_location":{"id":"doi:10.1017/s1351324909005051","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s1351324909005051","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039621382","display_name":"Rebecca J. Passonneau","orcid":"https://orcid.org/0000-0001-8626-811X"},"institutions":[{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"REBECCA J. PASSONNEAU","raw_affiliation_strings":["Center for Computational Learning Systems, Columbia University, NY 10115, USA e-mail:","Center for computational learning systems, columbia university, ny 10115, usa e-mail: becky@cs.columbia.edu#TAB#"],"affiliations":[{"raw_affiliation_string":"Center for Computational Learning Systems, Columbia University, NY 10115, USA e-mail:","institution_ids":["https://openalex.org/I78577930"]},{"raw_affiliation_string":"Center for computational learning systems, columbia university, ny 10115, usa e-mail: becky@cs.columbia.edu#TAB#","institution_ids":["https://openalex.org/I78577930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5039621382"],"corresponding_institution_ids":["https://openalex.org/I78577930"],"apc_list":null,"apc_paid":null,"fwci":2.1809,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.89255725,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"16","issue":"2","first_page":"107","last_page":"131"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pyramid","display_name":"Pyramid (geometry)","score":0.89364093542099},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8406988382339478},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.783887505531311},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6891956925392151},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.6421384215354919},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5192601680755615},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.440498411655426},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.38726842403411865},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3650452196598053},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3534172773361206},{"id":"https://openalex.org/keywords/power","display_name":"Power (physics)","score":0.2808021903038025},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.2339111864566803},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0894206166267395}],"concepts":[{"id":"https://openalex.org/C142575187","wikidata":"https://www.wikidata.org/wiki/Q3358290","display_name":"Pyramid (geometry)","level":2,"score":0.89364093542099},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8406988382339478},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.783887505531311},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6891956925392151},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.6421384215354919},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5192601680755615},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.440498411655426},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38726842403411865},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3650452196598053},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3534172773361206},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.2808021903038025},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.2339111864566803},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0894206166267395},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1017/s1351324909005051","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s1351324909005051","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","display_name":"Reduced inequalities","score":0.6399999856948853}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W1481038076","https://openalex.org/W1489409710","https://openalex.org/W1495422091","https://openalex.org/W1501617060","https://openalex.org/W1520453943","https://openalex.org/W1530435204","https://openalex.org/W1552513054","https://openalex.org/W1744393362","https://openalex.org/W1747645360","https://openalex.org/W1977747299","https://openalex.org/W1985463960","https://openalex.org/W1987869189","https://openalex.org/W2033375979","https://openalex.org/W2053154970","https://openalex.org/W2068406960","https://openalex.org/W2078861931","https://openalex.org/W2079656678","https://openalex.org/W2085575316","https://openalex.org/W2098159102","https://openalex.org/W2101105183","https://openalex.org/W2102065370","https://openalex.org/W2103333826","https://openalex.org/W2111273940","https://openalex.org/W2123946480","https://openalex.org/W2128530885","https://openalex.org/W2134061542","https://openalex.org/W2137591918","https://openalex.org/W2140424253","https://openalex.org/W2140440594","https://openalex.org/W2145907631","https://openalex.org/W2149593800","https://openalex.org/W2153222072","https://openalex.org/W2153804780","https://openalex.org/W2160204597","https://openalex.org/W2161676054","https://openalex.org/W2169401582","https://openalex.org/W2612753051","https://openalex.org/W2916445322","https://openalex.org/W2950627632","https://openalex.org/W3021916629","https://openalex.org/W3138771162","https://openalex.org/W4210551050","https://openalex.org/W4231438834","https://openalex.org/W4240908132","https://openalex.org/W4285719527","https://openalex.org/W6667353755","https://openalex.org/W6675969814","https://openalex.org/W6680768466","https://openalex.org/W6682780409","https://openalex.org/W6792221166"],"related_works":["https://openalex.org/W2361861616","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2358755282","https://openalex.org/W2625833328","https://openalex.org/W1533177136","https://openalex.org/W2251519152","https://openalex.org/W4380994516"],"abstract_inverted_index":{"Abstract":[0],"Pyramid":[1],"annotation":[2,57,78,83],"makes":[3],"it":[4],"possible":[5],"to":[6,145,174,192],"evaluate":[7],"quantitatively":[8],"and":[9,60,82,91,126,162,199],"qualitatively":[10],"the":[11,24,43,54,75,95,132,147,150,171,182,186,201,207],"content":[12],"of":[13,39,56,70,74,84,94,97,112,115,124,137,149,164,185],"machine-generated":[14],"(or":[15],"human)":[16],"summaries.":[17,166],"Evaluation":[18],"methods":[19],"must":[20],"prove":[21],"themselves":[22],"against":[23,88],"same":[25,133],"measuring":[26],"stick":[27],"\u2013":[28,30],"evaluation":[29,154],"as":[31],"other":[32],"research":[33],"methods.":[34],"First,":[35],"a":[36,113],"formal":[37],"assessment":[38,169],"pyramid":[40,89,125],"data":[41,139],"from":[42,100,120],"2003":[44],"Document":[45],"Understanding":[46],"Conference":[47],"(DUC)":[48],"is":[49,58,188],"presented;":[50],"this":[51],"addresses":[52,170],"whether":[53,61],"form":[55],"reliable":[59,105],"score":[62],"results":[63],"are":[64,143],"consistent":[65],"across":[66,152,177,206],"annotators.":[67],"A":[68],"combination":[69],"interannotator":[71],"reliability":[72,148],"measures":[73],"two":[76,121],"manual":[77],"phases":[79],"(pyramid":[80],"creation":[81],"system":[85,98,117],"peer":[86,116,127],"summaries":[87],"models),":[90],"significance":[92],"tests":[93],"similarity":[96],"scores":[99],"distinct":[101,153,156],"annotations,":[102,128],"produces":[103],"highly":[104],"results.":[106],"The":[107],"most":[108],"rigorous":[109],"test":[110],"consists":[111],"comparison":[114],"rankings":[118],"produced":[119],"independent":[122],"sets":[123],"which":[129],"produce":[130],"essentially":[131],"rankings.":[134],"Three":[135],"years":[136],"DUC":[138],"(2003,":[140],"2005,":[141],"2006)":[142],"used":[144],"assess":[146],"method":[151,187],"settings:":[155],"systems,":[157,198],"document":[158],"sets,":[159],"summary":[160],"lengths,":[161],"numbers":[163],"model":[165],"This":[167],"functional":[168],"method's":[172],"ability":[173],"discriminate":[175],"systems":[176],"years.":[178,209],"Results":[179],"indicate":[180],"that":[181,200],"statistical":[183,202],"power":[184,203],"more":[189],"than":[190],"sufficient":[191],"identify":[193],"statistically":[194],"significant":[195],"differences":[196],"among":[197],"varies":[204],"little":[205],"3":[208]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":4},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":1},{"year":2016,"cited_by_count":2},{"year":2013,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}