Feature/add meli search

README.md

@@ -49,7 +49,7 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d
 In the Wiki there are also examples for: a [manual installation](https://github.com/janeczku/calibre-web/wiki/Manual-installation), [installation on Linux Mint](https://github.com/janeczku/calibre-web/wiki/How-To:Install-Calibre-Web-in-Linux-Mint-19-or-20), [installation on a Cloud Provider](https://github.com/janeczku/calibre-web/wiki/How-To:-Install-Calibre-Web-on-a-Cloud-Provider).
 
 ## Quick start
-
+Install `freetype imagemagick` in your os
 Point your browser to `http://localhost:8083` or `http://localhost:8083/opds` for the OPDS catalog \
 Login with default admin login \
 Set `Location of Calibre database` to the path of the folder where your Calibre library (metadata.db) lives, push "submit" button \

cps/admin.py

@@ -106,6 +106,9 @@ def before_request():
         confirm_login()
     if not ub.check_user_session(current_user.id, flask_session.get('_id')) and 'opds' not in request.path:
         logout_user()
+    g.meili_host = os.environ["MEILI_HOST"]
+    g.api_key = os.environ["MEILI_KEY"]
+
     g.constants = constants
     g.user = current_user
     g.allow_registration = config.config_public_reg

cps/db.py

@@ -23,6 +23,7 @@
 from datetime import datetime
 from urllib.parse import quote
 import unidecode
+from pydantic import BaseModel
 
 from sqlite3 import OperationalError as sqliteOperationalError
 from sqlalchemy import create_engine
@@ -47,6 +48,7 @@
 
 from . import logger, ub, isoLanguages
 from .pagination import Pagination
+from .meilie_db import BookSearch
 
 from weakref import WeakSet
 
@@ -361,7 +363,9 @@ class Books(Base):
     identifiers = relationship(Identifiers, backref='books')
 
     def __init__(self, title, sort, author_sort, timestamp, pubdate, series_index, last_modified, path, has_cover,
-                 authors, tags, languages=None):
+                 authors=None, tags=None, languages=None, id=None, **kwargs):
+        if id:
+            self.id = id
         self.title = title
         self.sort = sort
         self.author_sort = author_sort
@@ -381,6 +385,10 @@ def __repr__(self):
     def atom_timestamp(self):
         return self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or ''
 
+class PayloadBook(BaseModel):
+    Books: Books
+    class Config:
+        arbitrary_types_allowed = True
 
 class CustomColumns(Base):
     __tablename__ = 'custom_columns'
@@ -479,6 +487,7 @@ def __init__(self, expire_on_commit=True, init=False):
         self.session = None
         if init:
             self.init_db(expire_on_commit)
+        self.book_search = BookSearch()
 
 
     def init_db(self, expire_on_commit=True):
@@ -928,20 +937,21 @@ def get_cc_columns(self, config, filter_config_custom_read=False):
     def get_search_results(self, term, config, offset=None, order=None, limit=None, *join):
         order = order[0] if order else [Books.sort]
         pagination = None
-        result = self.search_query(term, config, *join).order_by(*order).all()
+        result = self.book_search.search(term=term, config=config)
+        result = [PayloadBook(Books = Books(**i)) for i in result]
         result_count = len(result)
         if offset != None and limit != None:
             offset = int(offset)
-            limit_all = offset + int(limit)
+            # limit_all = offset + int(limit)
             pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
         else:
             offset = 0
-            limit_all = result_count
+            # limit_all = result_count
 
-        ub.store_combo_ids(result)
-        entries = self.order_authors(result[offset:limit_all], list_return=True, combined=True)
+        # ub.store_combo_ids(result)
+        # entries = self.order_authors(result[offset:limit_all], list_return=True, combined=True)
 
-        return entries, result_count, pagination
+        return result, result_count, pagination
 
     # Creates for all stored languages a translated speaking name in the array for the UI
     def speaking_language(self, languages=None, return_all_languages=False, with_count=False, reverse_order=False):
@@ -1041,6 +1051,12 @@ def lcase(s):
         _log.error_or_exception(ex)
         return s.lower()
 
+def row2dict(row):
+    d = {}
+    for column in row.__table__.columns:
+        d[column.name] = str(getattr(row, column.name))
+
+    return d
 
 class Category:
     name = None

cps/editbooks.py

@@ -48,6 +48,7 @@
 from .render_template import render_title_template
 from .usermanagement import login_required_if_no_ano
 from .kobo_sync_status import change_archived_books
+from .meilie_db import BookSearch
 
 
 editbook = Blueprint('edit-book', __name__)
@@ -207,6 +208,8 @@ def edit_book(book_id):
 
         calibre_db.session.merge(book)
         calibre_db.session.commit()
+        book_meili = BookSearch()
+        book_meili.insert_book(db.row2dict(book))
         if config.config_use_google_drive:
             gdriveutils.updateGdriveCalibreFromLocal()
         if meta is not False \
@@ -242,6 +245,7 @@ def upload():
     if not config.config_uploading:
         abort(404)
     if request.method == 'POST' and 'btn-upload' in request.files:
+        book_meili = BookSearch()
         for requested_file in request.files.getlist("btn-upload"):
             try:
                 modify_date = False
@@ -260,6 +264,9 @@ def upload():
 
                 book_id = db_book.id
                 title = db_book.title
+
+                book_dict = db.row2dict(db_book)
+                book_meili.insert_book(db_book=book_dict)
                 if config.config_use_google_drive:
                     helper.upload_new_file_gdrive(book_id,
                                                   input_authors[0],
@@ -806,6 +813,8 @@ def delete_whole_book(book_id, book):
             modify_database_object([u''], getattr(book, cc_string), db.cc_classes[c.id],
                                    calibre_db.session, 'custom')
     calibre_db.session.query(db.Books).filter(db.Books.id == book_id).delete()
+    book_meili = BookSearch()
+    book_meili.delete_book(book_id=book_id)
 
 
 def render_delete_book_result(book_format, json_response, warning, book_id):

cps/epub.py

@@ -25,23 +25,128 @@
 from .constants import BookMeta
 
 
-def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name):
-    if cover_file is None:
+def _extract_cover(zip_file, cover_path, tmp_file_name):
+    if cover_path is None:
         return None
     else:
         cf = extension = None
-        zip_cover_path = os.path.join(cover_path, cover_file).replace('\\', '/')
 
         prefix = os.path.splitext(tmp_file_name)[0]
-        tmp_cover_name = prefix + '.' + os.path.basename(zip_cover_path)
+        tmp_cover_name = prefix + '.' + os.path.basename(cover_path)
         ext = os.path.splitext(tmp_cover_name)
         if len(ext) > 1:
             extension = ext[1].lower()
         if extension in cover.COVER_EXTENSIONS:
-            cf = zip_file.read(zip_cover_path)
+            cf = zip_file.read(cover_path)
         return cover.cover_processing(tmp_file_name, cf, extension)
 
 
+def get_epub_cover(zipfile):
+    namespaces = {
+   "calibre":"http://calibre.kovidgoyal.net/2009/metadata",
+   "dc":"http://purl.org/dc/elements/1.1/",
+   "dcterms":"http://purl.org/dc/terms/",
+   "opf":"http://www.idpf.org/2007/opf",
+   "u":"urn:oasis:names:tc:opendocument:xmlns:container",
+   "xsi":"http://www.w3.org/2001/XMLSchema-instance",
+   "xhtml":"http://www.w3.org/1999/xhtml"
+}
+    t = etree.fromstring(zipfile.read("META-INF/container.xml"))
+        # We use xpath() to find the attribute "full-path":
+    '''
+    <container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
+        <rootfiles>
+        <rootfile full-path="OEBPS/content.opf" ... />
+        </rootfiles>
+    </container>
+    '''
+    rootfile_path =  t.xpath("/u:container/u:rootfiles/u:rootfile",
+                                            namespaces=namespaces)[0].get("full-path")
+
+    # We load the "root" file, indicated by the "full_path" attribute of "META-INF/container.xml", using lxml.etree.fromString():
+    t = etree.fromstring(zipfile.read(rootfile_path))
+
+    cover_href = None
+    try:
+        # For EPUB 2.0, we use xpath() to find a <meta> 
+        # named "cover" and get the attribute "content":
+        '''
+        <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
+            ...
+            <meta content="my-cover-image" name="cover"/>
+            ...
+        </metadata>            '''
+
+        cover_id = t.xpath("//opf:metadata/opf:meta[@name='cover']",
+                                    namespaces=namespaces)[0].get("content")
+        # Next, we use xpath() to find the <item> (in <manifest>) with this id 
+        # and get the attribute "href":
+        '''
+        <manifest>
+            ...
+            <item id="my-cover-image" href="images/978.jpg" ... />
+            ... 
+        </manifest>
+        '''
+        cover_href = t.xpath("//opf:manifest/opf:item[@id='" + cover_id + "']",
+                                namespaces=namespaces)[0].get("href")
+    except IndexError:
+        pass
+
+    if not cover_href:
+        # For EPUB 3.0, We use xpath to find the <item> (in <manifest>) that
+        # has properties='cover-image' and get the attribute "href":
+        '''
+        <manifest>
+            ...
+            <item href="images/cover.png" id="cover-img" media-type="image/png" properties="cover-image"/>
+            ...
+        </manifest>
+        '''
+        try:
+            cover_href = t.xpath("//opf:manifest/opf:item[@properties='cover-image']",
+                                    namespaces=namespaces)[0].get("href")
+        except IndexError:
+            pass
+
+    if not cover_href:
+        # Some EPUB files do not declare explicitly a cover image.
+        # Instead, they use an "<img src=''>" inside the first xhmtl file.
+        try:
+            # The <spine> is a list that defines the linear reading order
+            # of the content documents of the book. The first item in the  
+            # list is the first item in the book.  
+            '''
+            <spine toc="ncx">
+                <itemref idref="cover"/>
+                <itemref idref="nav"/>
+                <itemref idref="s04"/>
+            </spine>
+            '''
+            cover_page_id = t.xpath("//opf:spine/opf:itemref",
+                                    namespaces=namespaces)[0].get("idref")
+            # Next, we use xpath() to find the item (in manifest) with this id 
+            # and get the attribute "href":
+            cover_page_href = t.xpath("//opf:manifest/opf:item[@id='" + cover_page_id + "']",
+                                        namespaces=namespaces)[0].get("href")
+            # In order to get the full path for the cover page,
+            # we have to join rootfile_path and cover_page_href:
+            cover_page_path = os.path.join(os.path.dirname(rootfile_path), cover_page_href)
+            # We try to find the <img> and get the "src" attribute:
+            t = etree.fromstring(zipfile.read(cover_page_path))              
+            cover_href = t.xpath("//xhtml:img", namespaces=namespaces)[0].get("src")
+        except IndexError:
+            pass
+
+    if not cover_href:
+        return None
+
+    # In order to get the full path for the cover image,
+    # we have to join rootfile_path and cover_href:
+    cover_href = cover_href.replace("../", "")
+    cover_path = os.path.join(os.path.dirname(rootfile_path), cover_href)
+    return cover_path
+
 def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     ns = {
         'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
@@ -57,7 +162,6 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     cf = epub_zip.read(cf_name)
     tree = etree.fromstring(cf)
 
-    cover_path = os.path.dirname(cf_name)
 
     p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0]
 
@@ -98,7 +202,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
 
     epub_metadata = parse_epub_series(ns, tree, epub_metadata)
 
-    cover_file = parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path)
+    cover_file = parse_epub_cover(epub_zip, tmp_file_path)
 
     identifiers = []
     for node in p.xpath('dc:identifier', namespaces=ns):
@@ -129,42 +233,12 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
         identifiers=identifiers)
 
 
-def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):
-    cover_section = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
-    cover_file = None
+def parse_epub_cover(epub_zip, tmp_file_path):
+    cover_file = get_epub_cover(zipfile=epub_zip)
     # if len(cover_section) > 0:
-    for cs in cover_section:
-        cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
-        if cover_file:
-            break
-    if not cover_file:
-        meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns)
-        if len(meta_cover) > 0:
-            cover_section = tree.xpath(
-                "/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns)
-            if not cover_section:
-                cover_section = tree.xpath(
-                    "/pkg:package/pkg:manifest/pkg:item[@properties='" + meta_cover[0] + "']/@href", namespaces=ns)
-        else:
-            cover_section = tree.xpath("/pkg:package/pkg:guide/pkg:reference/@href", namespaces=ns)
-        for cs in cover_section:
-            filetype = cs.rsplit('.', 1)[-1]
-            if filetype == "xhtml" or filetype == "html":  # if cover is (x)html format
-                markup = epub_zip.read(os.path.join(cover_path, cs))
-                markup_tree = etree.fromstring(markup)
-                # no matter xhtml or html with no namespace
-                img_src = markup_tree.xpath("//*[local-name() = 'img']/@src")
-                # Alternative image source
-                if not len(img_src):
-                    img_src = markup_tree.xpath("//attribute::*[contains(local-name(), 'href')]")
-                if len(img_src):
-                    # img_src maybe start with "../"" so fullpath join then relpath to cwd
-                    filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(cover_path, cover_section[0])),
-                                                            img_src[0]))
-                    cover_file = _extract_cover(epub_zip, filename, "", tmp_file_path)
-            else:
-                cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
-            if cover_file: break
+    if cover_file:
+        cover_file = _extract_cover(epub_zip, cover_file, tmp_file_path)
+
     return cover_file
 
 

cps/logger.py

@@ -43,7 +43,7 @@
 class _Logger(logging.Logger):
 
     def error_or_exception(self, message, stacklevel=2, *args, **kwargs):
-        if sys.version_info > (3, 7):
+        if sys.version_info.minor > 7:
             if is_debug_enabled():
                 self.exception(message, stacklevel=stacklevel, *args, **kwargs)
             else: