From 5edde53fedbe54137d95d2b8a30bd92391ed890c Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 3 Oct 2021 09:53:46 +0200 Subject: [PATCH] Improved sync for kobo with additional table --- cps/kobo.py | 50 ++++++++++++++++++++++++++------------- cps/services/SyncToken.py | 26 ++++++++++---------- cps/ub.py | 6 +++++ 3 files changed, 53 insertions(+), 29 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index 6952a6928..7eee3e17d 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -170,9 +170,14 @@ def HandleSyncRequest(): ub.ArchivedBook.is_archived) changed_entries = (changed_entries .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id) - .filter(or_(db.Books.last_modified > sync_token.books_last_modified, - ub.BookShelf.date_added > sync_token.books_last_modified)) - .filter(db.Data.format.in_(KOBO_FORMATS)).filter(calibre_db.common_filters()) + .join(ub.KoboSyncedBooks, ub.KoboSyncedBooks.book_id == db.Books.id, isouter=True) + .filter(or_(ub.KoboSyncedBooks.user_id != current_user.id, + ub.KoboSyncedBooks.book_id == None)) + #.filter(or_(db.Books.last_modified > sync_token.books_last_modified, + # ub.BookShelf.date_added > sync_token.books_last_modified)) + .filter(ub.BookShelf.date_added > sync_token.books_last_modified) #?? or also or from above + .filter(db.Data.format.in_(KOBO_FORMATS)) + .filter(calibre_db.common_filters()) .order_by(db.Books.id) .order_by(ub.ArchivedBook.last_modified) .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id) @@ -189,16 +194,18 @@ def HandleSyncRequest(): ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived) changed_entries = (changed_entries - .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id) - .filter(db.Books.last_modified > sync_token.books_last_modified) - .filter(calibre_db.common_filters()) - .filter(db.Data.format.in_(KOBO_FORMATS)) - .order_by(db.Books.last_modified) - .order_by(db.Books.id) + .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id) + .join(ub.KoboSyncedBooks, ub.KoboSyncedBooks.book_id == db.Books.id, isouter=True) + .filter(or_(ub.KoboSyncedBooks.user_id != current_user.id, + ub.KoboSyncedBooks.book_id == None)) + .filter(calibre_db.common_filters()) + .filter(db.Data.format.in_(KOBO_FORMATS)) + .order_by(db.Books.last_modified) + .order_by(db.Books.id) ) - if sync_token.books_last_id > -1: - changed_entries = changed_entries.filter(db.Books.id > sync_token.books_last_id) + #if sync_token.books_last_id > -1: + # changed_entries = changed_entries.filter(db.Books.id > sync_token.books_last_id) reading_states_in_new_entitlements = [] if sqlalchemy_version2: @@ -206,6 +213,7 @@ def HandleSyncRequest(): else: books = changed_entries.limit(SYNC_ITEM_LIMIT) for book in books: + add_synced_books(book.Books.id) formats = [data.format for data in book.Books.data] if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats: helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name) @@ -263,11 +271,11 @@ def HandleSyncRequest(): entries = calibre_db.session.execute(changed_entries).all() book_count = len(entries) else: - entries = changed_entries.all() + #entries = changed_entries.all() book_count = changed_entries.count() # last entry: - books_last_id = entries[-1].Books.id or -1 if book_count else -1 - + # sync_cont = entries[-1].Books.id or -1 if book_count else -1 + log.debug("Remaining books to Sync: {}".format(book_count)) # generate reading state data changed_reading_states = ub.session.query(ub.KoboReadingState) @@ -305,7 +313,7 @@ def HandleSyncRequest(): sync_token.books_last_modified = new_books_last_modified sync_token.archive_last_modified = new_archived_last_modified sync_token.reading_state_last_modified = new_reading_state_last_modified - sync_token.books_last_id = books_last_id + # sync_token.books_last_id = books_last_id return generate_sync_response(sync_token, sync_results, book_count) @@ -330,7 +338,7 @@ def generate_sync_response(sync_token, sync_results, set_cont=False): extra_headers["x-kobo-sync"] = "continue" sync_token.to_headers(extra_headers) - log.debug("Kobo Sync Content: {}".format(sync_results)) + # log.debug("Kobo Sync Content: {}".format(sync_results)) response = make_response(jsonify(sync_results), extra_headers) return response @@ -838,6 +846,16 @@ def get_ub_read_status(kobo_read_status): } return string_to_enum_map[kobo_read_status] +def add_synced_books(book_id): + synced_book = ub.KoboSyncedBooks() + synced_book.user_id = current_user.id + synced_book.book_id = book_id + ub.session.add(synced_book) + try: + ub.session.commit() + except Exception: + ub.session.rollback() + def get_or_create_reading_state(book_id): book_read = ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id, diff --git a/cps/services/SyncToken.py b/cps/services/SyncToken.py index cc67542c3..86165f717 100644 --- a/cps/services/SyncToken.py +++ b/cps/services/SyncToken.py @@ -85,8 +85,8 @@ class SyncToken: "books_last_created": {"type": "string"}, "archive_last_modified": {"type": "string"}, "reading_state_last_modified": {"type": "string"}, - "tags_last_modified": {"type": "string"}, - "books_last_id": {"type": "integer", "optional": True} + "tags_last_modified": {"type": "string"} + # "books_last_id": {"type": "integer", "optional": True} }, } @@ -97,8 +97,8 @@ def __init__( books_last_modified=datetime.min, archive_last_modified=datetime.min, reading_state_last_modified=datetime.min, - tags_last_modified=datetime.min, - books_last_id=-1 + tags_last_modified=datetime.min + # books_last_id=-1 ): # nosec self.raw_kobo_store_token = raw_kobo_store_token self.books_last_created = books_last_created @@ -106,7 +106,7 @@ def __init__( self.archive_last_modified = archive_last_modified self.reading_state_last_modified = reading_state_last_modified self.tags_last_modified = tags_last_modified - self.books_last_id = books_last_id + # self.books_last_id = books_last_id @staticmethod def from_headers(headers): @@ -141,12 +141,12 @@ def from_headers(headers): archive_last_modified = get_datetime_from_json(data_json, "archive_last_modified") reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified") tags_last_modified = get_datetime_from_json(data_json, "tags_last_modified") - books_last_id = data_json["books_last_id"] + # books_last_id = data_json["books_last_id"] except TypeError: log.error("SyncToken timestamps don't parse to a datetime.") return SyncToken(raw_kobo_store_token=raw_kobo_store_token) - except KeyError: - books_last_id = -1 + #except KeyError: + # books_last_id = -1 return SyncToken( raw_kobo_store_token=raw_kobo_store_token, @@ -155,7 +155,7 @@ def from_headers(headers): archive_last_modified=archive_last_modified, reading_state_last_modified=reading_state_last_modified, tags_last_modified=tags_last_modified, - books_last_id=books_last_id + #books_last_id=books_last_id ) def set_kobo_store_header(self, store_headers): @@ -179,16 +179,16 @@ def build_sync_token(self): "archive_last_modified": to_epoch_timestamp(self.archive_last_modified), "reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified), "tags_last_modified": to_epoch_timestamp(self.tags_last_modified), - "books_last_id":self.books_last_id + #"books_last_id":self.books_last_id }, } return b64encode_json(token) def __str__(self): - return "{},{},{},{},{},{},{}".format(self.raw_kobo_store_token, + return "{},{},{},{},{},{}".format(self.raw_kobo_store_token, self.books_last_created, self.books_last_modified, self.archive_last_modified, self.reading_state_last_modified, - self.tags_last_modified, - self.books_last_id) + self.tags_last_modified) + #self.books_last_id) diff --git a/cps/ub.py b/cps/ub.py index f1a33d756..df1f0bc81 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -419,6 +419,12 @@ class ArchivedBook(Base): last_modified = Column(DateTime, default=datetime.datetime.utcnow) +class KoboSyncedBooks(Base): + __tablename__ = 'kobo_synced_books' + id = Column(Integer, primary_key=True, autoincrement=True) + user_id = Column(Integer, ForeignKey('user.id')) + book_id = Column(Integer) + # The Kobo ReadingState API keeps track of 4 timestamped entities: # ReadingState, StatusInfo, Statistics, CurrentBookmark # Which we map to the following 4 tables: