Merge pull request #602 from kiwix/partial_opds_entries

OPDS feed with partial entries
This commit is contained in:
Matthieu Gautier 2021-09-09 12:07:02 +02:00 committed by GitHub
commit 3b942bb745
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 196 additions and 90 deletions

View File

@ -59,9 +59,18 @@ class OPDSDumper
*
* @param bookIds the ids of the books to include in the feed
* @param query the query used to obtain the list of book ids
* @param partial whether the feed should include partial or complete entries
* @return The OPDS feed.
*/
std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const;
std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const;
/**
* Dump the OPDS complete entry document.
*
* @param bookId the id of the book
* @return The OPDS complete entry document.
*/
std::string dumpOPDSCompleteEntry(const std::string& bookId) const;
/**
* Dump the categories OPDS feed.

View File

@ -51,7 +51,7 @@ namespace
{
typedef kainjow::mustache::data MustacheData;
typedef kainjow::mustache::list BookData;
typedef kainjow::mustache::list BooksData;
typedef kainjow::mustache::list IllustrationInfo;
IllustrationInfo getBookIllustrationInfo(const Book& book)
@ -69,16 +69,13 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)
return illustrations;
}
BookData getBookData(const Library* library, const std::vector<std::string>& bookIds)
kainjow::mustache::object getSingleBookData(const Book& book)
{
BookData bookData;
for ( const auto& bookId : bookIds ) {
const Book& book = library->getBookById(bookId);
const MustacheData bookUrl = book.getUrl().empty()
? MustacheData(false)
: MustacheData(book.getUrl());
bookData.push_back(kainjow::mustache::object{
{"id", "urn:uuid:"+book.getId()},
return kainjow::mustache::object{
{"id", book.getId()},
{"name", book.getName()},
{"title", book.getTitle()},
{"description", book.getDescription()},
@ -95,10 +92,29 @@ BookData getBookData(const Library* library, const std::vector<std::string>& boo
{"url", bookUrl},
{"size", to_string(book.getSize())},
{"icons", getBookIllustrationInfo(book)},
};
}
std::string getSingleBookEntryXML(const Book& book, bool withXMLHeader, const std::string& endpointRoot, bool partial)
{
auto data = getSingleBookData(book);
data["with_xml_header"] = MustacheData(withXMLHeader);
data["dump_partial_entries"] = MustacheData(partial);
data["endpoint_root"] = endpointRoot;
return render_template(RESOURCE::templates::catalog_v2_entry_xml, data);
}
BooksData getBooksData(const Library* library, const std::vector<std::string>& bookIds, const std::string& endpointRoot, bool partial)
{
BooksData booksData;
for ( const auto& bookId : bookIds ) {
const Book& book = library->getBookById(bookId);
booksData.push_back(kainjow::mustache::object{
{"entry", getSingleBookEntryXML(book, false, endpointRoot, partial)}
});
}
return bookData;
return booksData;
}
std::string getLanguageSelfName(const std::string& lang) {
@ -114,7 +130,7 @@ std::string getLanguageSelfName(const std::string& lang) {
string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const
{
const auto bookData = getBookData(library, bookIds);
const auto booksData = getBooksData(library, bookIds, "", false);
const kainjow::mustache::object template_data{
{"date", gen_date_str()},
{"root", rootLocation},
@ -123,31 +139,39 @@ string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const s
{"totalResults", to_string(m_totalResults)},
{"startIndex", to_string(m_startIndex)},
{"itemsPerPage", to_string(m_count)},
{"books", bookData }
{"books", booksData }
};
return render_template(RESOURCE::templates::catalog_entries_xml, template_data);
}
string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const
string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query, bool partial) const
{
const auto bookData = getBookData(library, bookIds);
const auto endpointRoot = rootLocation + "/catalog/v2";
const auto booksData = getBooksData(library, bookIds, endpointRoot, partial);
const char* const endpoint = partial ? "/partial_entries" : "/entries";
const kainjow::mustache::object template_data{
{"date", gen_date_str()},
{"endpoint_root", rootLocation + "/catalog/v2"},
{"feed_id", gen_uuid(libraryId + "/entries?"+query)},
{"endpoint_root", endpointRoot},
{"feed_id", gen_uuid(libraryId + endpoint + "?" + query)},
{"filter", query.empty() ? MustacheData(false) : MustacheData(query)},
{"query", query.empty() ? "" : "?" + urlEncode(query)},
{"totalResults", to_string(m_totalResults)},
{"startIndex", to_string(m_startIndex)},
{"itemsPerPage", to_string(m_count)},
{"books", bookData }
{"books", booksData },
{"dump_partial_entries", MustacheData(partial)}
};
return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data);
}
std::string OPDSDumper::dumpOPDSCompleteEntry(const std::string& bookId) const
{
return getSingleBookEntryXML(library->getBookById(bookId), true, "", false);
}
std::string OPDSDumper::categoriesOPDSFeed() const
{
const auto now = gen_date_str();

View File

@ -75,7 +75,8 @@ class InternalServer {
std::unique_ptr<Response> handle_catalog(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_root(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_entries(const RequestContext& request, bool partial);
std::unique_ptr<Response> handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId);
std::unique_ptr<Response> handle_catalog_v2_categories(const RequestContext& request);
std::unique_ptr<Response> handle_catalog_v2_languages(const RequestContext& request);
std::unique_ptr<Response> handle_meta(const RequestContext& request);

View File

@ -55,8 +55,13 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
kainjow::mustache::object({{"endpoint_root", endpoint_root}}),
"application/opensearchdescription+xml"
);
} else if (url == "entry") {
const std::string entryId = request.get_url_part(3);
return handle_catalog_v2_complete_entry(request, entryId);
} else if (url == "entries") {
return handle_catalog_v2_entries(request);
return handle_catalog_v2_entries(request, /*partial=*/false);
} else if (url == "partial_entries") {
return handle_catalog_v2_entries(request, /*partial=*/true);
} else if (url == "categories") {
return handle_catalog_v2_categories(request);
} else if (url == "languages") {
@ -76,6 +81,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestCo
{"endpoint_root", m_root + "/catalog/v2"},
{"feed_id", gen_uuid(m_library_id)},
{"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
{"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
{"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
{"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
},
@ -83,13 +89,13 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestCo
);
}
std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request)
std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const RequestContext& request, bool partial)
{
OPDSDumper opdsDumper(mp_library);
opdsDumper.setRootLocation(m_root);
opdsDumper.setLibraryId(m_library_id);
const auto bookIds = search_catalog(request, opdsDumper);
const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query());
const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
return ContentResponse::build(
*this,
opdsFeed,
@ -97,6 +103,25 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const Reques
);
}
std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const RequestContext& request, const std::string& entryId)
{
try {
mp_library->getBookById(entryId);
} catch (const std::out_of_range&) {
return Response::build_404(*this, request, "", "");
}
OPDSDumper opdsDumper(mp_library);
opdsDumper.setRootLocation(m_root);
opdsDumper.setLibraryId(m_library_id);
const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
return ContentResponse::build(
*this,
opdsFeed,
"application/atom+xml;type=entry;profile=opds-catalog"
);
}
std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const RequestContext& request)
{
OPDSDumper opdsDumper(mp_library);

View File

@ -45,6 +45,7 @@ templates/captured_external.html
templates/catalog_entries.xml
templates/catalog_v2_root.xml
templates/catalog_v2_entries.xml
templates/catalog_v2_entry.xml
templates/catalog_v2_categories.xml
templates/catalog_v2_languages.xml
opensearchdescription.xml

View File

@ -9,34 +9,4 @@
{{/filter}}
<link rel="self" href="" type="application/atom+xml" />
<link rel="search" type="application/opensearchdescription+xml" href="{{root}}/catalog/searchdescription.xml" />
{{#books}}
<entry>
<id>{{id}}</id>
<title>{{title}}</title>
<summary>{{description}}</summary>
<language>{{language}}</language>
<updated>{{updated}}</updated>
<name>{{name}}</name>
<flavour>{{flavour}}</flavour>
<category>{{category}}</category>
<tags>{{tags}}</tags>
<articleCount>{{article_count}}</articleCount>
<mediaCount>{{media_count}}</mediaCount>
{{#icons}}
<link rel="http://opds-spec.org/image/thumbnail"
href="/meta?name=Illustration_{{icon_width}}x{{icon_height}}@{{icon_scale}}&amp;content={{{content_id}}}"
type="image/png;width={{icon_width}};height={{icon_height}};scale={{icon_scale}}"/>
{{/icons}}
<link type="text/html" href="/{{{content_id}}}" />
<author>
<name>{{author_name}}</name>
</author>
<publisher>
<name>{{publisher_name}}</name>
</publisher>
{{#url}}
<link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
{{/url}}
</entry>
{{/books}}
</feed>
{{#books}}{{{entry}}}{{/books}}</feed>

View File

@ -5,7 +5,7 @@
<id>{{feed_id}}</id>
<link rel="self"
href="{{endpoint_root}}/entries{{{query}}}"
href="{{endpoint_root}}/{{#dump_partial_entries}}partial_{{/dump_partial_entries}}entries{{{query}}}"
type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
<link rel="start"
href="{{endpoint_root}}/root.xml"
@ -21,34 +21,4 @@
<startIndex>{{startIndex}}</startIndex>
<itemsPerPage>{{itemsPerPage}}</itemsPerPage>
{{/filter}}
{{#books}}
<entry>
<id>{{id}}</id>
<title>{{title}}</title>
<summary>{{description}}</summary>
<language>{{language}}</language>
<updated>{{updated}}</updated>
<name>{{name}}</name>
<flavour>{{flavour}}</flavour>
<category>{{category}}</category>
<tags>{{tags}}</tags>
<articleCount>{{article_count}}</articleCount>
<mediaCount>{{media_count}}</mediaCount>
{{#icons}}
<link rel="http://opds-spec.org/image/thumbnail"
href="/meta?name=Illustration_{{icon_width}}x{{icon_height}}@{{icon_scale}}&amp;content={{{content_id}}}"
type="image/png;width={{icon_width}};height={{icon_height}};scale={{icon_scale}}"/>
{{/icons}}
<link type="text/html" href="/{{{content_id}}}" />
<author>
<name>{{author_name}}</name>
</author>
<publisher>
<name>{{publisher_name}}</name>
</publisher>
{{#url}}
<link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
{{/url}}
</entry>
{{/books}}
</feed>
{{#books}}{{{entry}}}{{/books}}</feed>

View File

@ -0,0 +1,34 @@
{{#with_xml_header}}<?xml version="1.0" encoding="UTF-8"?>
{{/with_xml_header}} <entry>
<id>urn:uuid:{{id}}</id>
<title>{{title}}</title>
<updated>{{updated}}</updated>
{{#dump_partial_entries}}
<link rel="alternate"
href="{{endpoint_root}}/entry/{{{id}}}"
type="application/atom+xml;type=entry;profile=opds-catalog"/>
{{/dump_partial_entries}}{{^dump_partial_entries}} <summary>{{description}}</summary>
<language>{{language}}</language>
<name>{{name}}</name>
<flavour>{{flavour}}</flavour>
<category>{{category}}</category>
<tags>{{tags}}</tags>
<articleCount>{{article_count}}</articleCount>
<mediaCount>{{media_count}}</mediaCount>
{{#icons}}
<link rel="http://opds-spec.org/image/thumbnail"
href="/meta?name=Illustration_{{icon_width}}x{{icon_height}}@{{icon_scale}}&amp;content={{{content_id}}}"
type="image/png;width={{icon_width}};height={{icon_height}};scale={{icon_scale}}"/>
{{/icons}}
<link type="text/html" href="/{{{content_id}}}" />
<author>
<name>{{author_name}}</name>
</author>
<publisher>
<name>{{publisher_name}}</name>
</publisher>
{{#url}}
<link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
{{/url}}
{{/dump_partial_entries}}
</entry>

View File

@ -23,6 +23,15 @@
<id>{{all_entries_feed_id}}</id>
<content type="text">All entries from this catalog.</content>
</entry>
<entry>
<title>All entries (partial)</title>
<link rel="subsection"
href="{{endpoint_root}}/partial_entries"
type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
<updated>{{date}}</updated>
<id>{{partial_entries_feed_id}}</id>
<content type="text">All entries from this catalog in partial format.</content>
</entry>
<entry>
<title>List of categories</title>
<link rel="subsection"

View File

@ -618,9 +618,9 @@ std::string maskVariableOPDSFeedData(std::string s)
" <entry>\n" \
" <id>urn:uuid:charlesray</id>\n" \
" <title>Charles, Ray</title>\n" \
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n" \
" <summary>Wikipedia articles about Ray Charles</summary>\n" \
" <language>fra</language>\n" \
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n" \
" <name>wikipedia_fr_ray_charles</name>\n" \
" <flavour></flavour>\n" \
" <category>jazz</category>\n" \
@ -644,9 +644,9 @@ std::string maskVariableOPDSFeedData(std::string s)
" <entry>\n" \
" <id>urn:uuid:raycharles</id>\n" \
" <title>Ray Charles</title>\n" \
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n" \
" <summary>Wikipedia articles about Ray Charles</summary>\n" \
" <language>eng</language>\n" \
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n" \
" <name>wikipedia_en_ray_charles</name>\n" \
" <flavour></flavour>\n" \
" <category>wikipedia</category>\n" \
@ -670,9 +670,9 @@ std::string maskVariableOPDSFeedData(std::string s)
" <entry>\n" \
" <id>urn:uuid:raycharles_uncategorized</id>\n" \
" <title>Ray (uncategorized) Charles</title>\n" \
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n" \
" <summary>No category is assigned to this library entry.</summary>\n" \
" <language>rus</language>\n" \
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n" \
" <name>wikipedia_ru_ray_charles</name>\n" \
" <flavour></flavour>\n" \
" <category></category>\n" \
@ -905,7 +905,6 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination)
" <startIndex>100</startIndex>\n"
" <itemsPerPage>0</itemsPerPage>\n"
CATALOG_LINK_TAGS
" \n"
"</feed>\n"
);
}
@ -940,6 +939,15 @@ TEST_F(LibraryServerTest, catalog_v2_root)
<id>12345678-90ab-cdef-1234-567890abcdef</id>
<content type="text">All entries from this catalog.</content>
</entry>
<entry>
<title>All entries (partial)</title>
<link rel="subsection"
href="/catalog/v2/partial_entries"
type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
<updated>YYYY-MM-DDThh:mm:ssZ</updated>
<id>12345678-90ab-cdef-1234-567890abcdef</id>
<content type="text">All entries from this catalog in partial format.</content>
</entry>
<entry>
<title>List of categories</title>
<link rel="subsection"
@ -1075,7 +1083,7 @@ TEST_F(LibraryServerTest, catalog_v2_languages)
EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output);
}
#define CATALOG_V2_ENTRIES_PREAMBLE(q) \
#define CATALOG_V2_ENTRIES_PREAMBLE0(x) \
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" \
"<feed xmlns=\"http://www.w3.org/2005/Atom\"\n" \
" xmlns:opds=\"https://specs.opds.io/opds-1.2\"\n" \
@ -1083,7 +1091,7 @@ TEST_F(LibraryServerTest, catalog_v2_languages)
" <id>12345678-90ab-cdef-1234-567890abcdef</id>\n" \
"\n" \
" <link rel=\"self\"\n" \
" href=\"/catalog/v2/entries" q "\"\n" \
" href=\"/catalog/v2/" x "\"\n" \
" type=\"application/atom+xml;profile=opds-catalog;kind=acquisition\"/>\n" \
" <link rel=\"start\"\n" \
" href=\"/catalog/v2/root.xml\"\n" \
@ -1093,6 +1101,11 @@ TEST_F(LibraryServerTest, catalog_v2_languages)
" type=\"application/atom+xml;profile=opds-catalog;kind=navigation\"/>\n" \
"\n" \
#define CATALOG_V2_ENTRIES_PREAMBLE(q) \
CATALOG_V2_ENTRIES_PREAMBLE0("entries" q)
#define CATALOG_V2_PARTIAL_ENTRIES_PREAMBLE(q) \
CATALOG_V2_ENTRIES_PREAMBLE0("partial_entries" q)
TEST_F(LibraryServerTest, catalog_v2_entries)
{
@ -1239,4 +1252,54 @@ TEST_F(LibraryServerTest, suggestions_in_range)
int currCount = std::count(body.begin(), body.end(), '{') - 1;
ASSERT_EQ(currCount, 0);
}
}
}
TEST_F(LibraryServerTest, catalog_v2_individual_entry_access)
{
const auto r = zfs1_->GET("/catalog/v2/entry/raycharles");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
RAY_CHARLES_CATALOG_ENTRY
);
const auto r1 = zfs1_->GET("/catalog/v2/entry/non-existent-entry");
EXPECT_EQ(r1->status, 404);
}
TEST_F(LibraryServerTest, catalog_v2_partial_entries)
{
const auto r = zfs1_->GET("/catalog/v2/partial_entries");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_PARTIAL_ENTRIES_PREAMBLE("")
" <title>All Entries</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
"\n"
" <entry>\n"
" <id>urn:uuid:charlesray</id>\n"
" <title>Charles, Ray</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <link rel=\"alternate\"\n"
" href=\"/catalog/v2/entry/charlesray\"\n"
" type=\"application/atom+xml;type=entry;profile=opds-catalog\"/>\n"
" </entry>\n"
" <entry>\n"
" <id>urn:uuid:raycharles</id>\n"
" <title>Ray Charles</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <link rel=\"alternate\"\n"
" href=\"/catalog/v2/entry/raycharles\"\n"
" type=\"application/atom+xml;type=entry;profile=opds-catalog\"/>\n"
" </entry>\n"
" <entry>\n"
" <id>urn:uuid:raycharles_uncategorized</id>\n"
" <title>Ray (uncategorized) Charles</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
" <link rel=\"alternate\"\n"
" href=\"/catalog/v2/entry/raycharles_uncategorized\"\n"
" type=\"application/atom+xml;type=entry;profile=opds-catalog\"/>\n"
" </entry>\n"
"</feed>\n"
);
}