Manually added books no longer removed on library refresh

Overhaul database module

TODO

@@ -73,8 +73,7 @@ TODO
         ✓ Define every widget in code
     Bugs:
         Slider position change might be acting up
-        Deleted directories should be removed from the database
+        Deselecting all directories in the settings dialog also filters out manually added books
-        There are still slashes in the code
 
     Secondary:
         Annotations

lector/__main__.py

@@ -302,7 +302,7 @@ class MainUI(QtWidgets.QMainWindow, mainwindow.Ui_MainWindow):
             file_list = [QtCore.QFileInfo(i).absoluteFilePath() for i in my_args]
             books = sorter.BookSorter(
                 file_list,
-                'addition',
+                ('addition', 'manual'),
                 self.database_path,
                 self.settings['auto_tags'],
                 self.temp_dir.path())
@@ -472,7 +472,7 @@ class MainUI(QtWidgets.QMainWindow, mainwindow.Ui_MainWindow):
         self.sorterProgress.setVisible(True)
         self.statusMessage.setText(self._translate('Main_UI', 'Adding books...'))
         self.thread = BackGroundBookAddition(
-            opened_files[0], self.database_path, False, self)
+            opened_files[0], self.database_path, 'manual', self)
         self.thread.finished.connect(self.move_on)
         self.thread.start()
 
@@ -756,7 +756,7 @@ class MainUI(QtWidgets.QMainWindow, mainwindow.Ui_MainWindow):
 
         contents = sorter.BookSorter(
             file_paths,
-            'reading',
+            ('reading', None),
             self.database_path,
             True,
             self.temp_dir.path()).initiate_threads()
@@ -1132,11 +1132,13 @@ class MainUI(QtWidgets.QMainWindow, mainwindow.Ui_MainWindow):
             checked = [i for i in directory_list if i[3] == QtCore.Qt.Checked]
             filter_list = list(map(generate_name, checked))
             filter_list.sort()
+
         filter_list.append(self._translate('Main_UI', 'Manually Added'))
         filter_actions = [QtWidgets.QAction(i, self.libraryFilterMenu) for i in filter_list]
 
         filter_all = QtWidgets.QAction('All', self.libraryFilterMenu)
         filter_actions.append(filter_all)
+
         for i in filter_actions:
             i.setCheckable(True)
             i.setChecked(True)
@@ -1166,6 +1168,7 @@ class MainUI(QtWidgets.QMainWindow, mainwindow.Ui_MainWindow):
         else:
             self.libraryFilterMenu.actions()[-1].setChecked(True)
 
+        # print(self.active_library_filters)
         self.lib_ref.update_proxymodels()
 
     def toggle_distraction_free(self):

lector/database.py

@@ -25,26 +25,67 @@ from PyQt5 import QtCore
 class DatabaseInit:
     def __init__(self, location_prefix):
         os.makedirs(location_prefix, exist_ok=True)
-        database_path = os.path.join(location_prefix, 'Lector.db')
+        self.database_path = os.path.join(location_prefix, 'Lector.db')
 
-        if not os.path.exists(database_path):
+        self.books_table_columns = {
-            self.database = sqlite3.connect(database_path)
+            'id': 'INTEGER PRIMARY KEY',
+            'Title': 'TEXT',
+            'Author': 'TEXT',
+            'Year': 'INTEGER',
+            'DateAdded': 'BLOB',
+            'Path': 'TEXT',
+            'Position': 'BLOB',
+            'ISBN': 'TEXT',
+            'Tags': 'TEXT',
+            'Hash': 'TEXT',
+            'LastAccessed': 'BLOB',
+            'Bookmarks': 'BLOB',
+            'CoverImage': 'BLOB',
+            'Addition': 'TEXT'}
+
+        self.directories_table_columns = {
+            'id': 'INTEGER PRIMARY KEY',
+            'Path': 'TEXT',
+            'Name': 'TEXT',
+            'Tags': 'TEXT',
+            'CheckState': 'INTEGER'}
+
+        if os.path.exists(self.database_path):
+            self.check_database()
+        else:
             self.create_database()
 
     def create_database(self):
-        # TODO
+        self.database = sqlite3.connect(self.database_path)
-        # Add separate columns for:
+
-        # addition mode
+        column_string = ', '.join(
-        self.database.execute(
+            [i[0] + ' ' + i[1] for i in self.books_table_columns.items()])
-            "CREATE TABLE books \
+        self.database.execute(f"CREATE TABLE books ({column_string})")
-            (id INTEGER PRIMARY KEY, Title TEXT, Author TEXT, Year INTEGER, DateAdded BLOB, \
-            Path TEXT, Position BLOB, ISBN TEXT, Tags TEXT, Hash TEXT, LastAccessed BLOB,\
-            Bookmarks BLOB, CoverImage BLOB)")
 
         # CheckState is the standard QtCore.Qt.Checked / Unchecked
-        self.database.execute(
+        column_string = ', '.join(
-            "CREATE TABLE directories (id INTEGER PRIMARY KEY, Path TEXT, \
+            [i[0] + ' ' + i[1] for i in self.directories_table_columns.items()])
-            Name TEXT, Tags TEXT, CheckState INTEGER)")
+        self.database.execute(f"CREATE TABLE directories ({column_string})")
+
+        self.database.commit()
+        self.database.close()
+
+    def check_database(self):
+        self.database = sqlite3.connect(self.database_path)
+
+        database_return = self.database.execute("PRAGMA table_info(books)").fetchall()
+        database_columns = [i[1] for i in database_return]
+
+        # This allows for addition of a column without having to reform the database
+        commit_required = False
+        for i in self.books_table_columns.items():
+            if i[0] not in database_columns:
+                commit_required = True
+                print(f'Database: Adding column {i[0]}')
+                sql_command = f"ALTER TABLE books ADD COLUMN {i[0]} {i[1]}"
+                self.database.execute(sql_command)
+
+        if commit_required:
             self.database.commit()
         self.database.close()
 
@@ -55,10 +96,6 @@ class DatabaseFunctions:
         self.database = sqlite3.connect(database_path)
 
     def set_library_paths(self, data_iterable):
-        # TODO
-        # INSERT OR REPLACE is not working
-        # So this is the old fashion kitchen sink approach
-
         self.database.execute("DELETE FROM directories")
 
         for i in data_iterable:
@@ -67,10 +104,13 @@ class DatabaseFunctions:
             tags = i[2]
             is_checked = i[3]
 
+            if not os.path.exists(path):
+                continue  # Remove invalid paths from the database
+
             sql_command = (
-                "INSERT OR REPLACE INTO directories (ID, Path, Name, Tags, CheckState)\
+                "INSERT INTO directories (Path, Name, Tags, CheckState)\
-                 VALUES ((SELECT ID FROM directories WHERE Path = ?), ?, ?, ?, ?)")
+                 VALUES (?, ?, ?, ?)")
-            self.database.execute(sql_command, [path, path, name, tags, is_checked])
+            self.database.execute(sql_command, [path, name, tags, is_checked])
 
         self.database.commit()
         self.database.close()
@@ -95,6 +135,7 @@ class DatabaseFunctions:
             path = i[1]['path']
             cover = i[1]['cover_image']
             isbn = i[1]['isbn']
+            addition_mode = i[1]['addition_mode']
             tags = i[1]['tags']
             if tags:
                 # Is a list. Needs to be a string
@@ -105,8 +146,9 @@ class DatabaseFunctions:
 
             sql_command_add = (
                 "INSERT OR REPLACE INTO \
-                books (Title, Author, Year, DateAdded, Path, ISBN, Tags, Hash, CoverImage) \
+                books (Title, Author, Year, DateAdded, Path, \
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
+                ISBN, Tags, Hash, CoverImage, Addition) \
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
 
             cover_insert = None
             if cover:
@@ -115,7 +157,8 @@ class DatabaseFunctions:
             self.database.execute(
                 sql_command_add,
                 [title, author, year, current_datetime_bin,
-                 path, isbn, tags, book_hash, cover_insert])
+                 path, isbn, tags, book_hash, cover_insert,
+                 addition_mode])
 
         self.database.commit()
         self.database.close()
@@ -208,9 +251,10 @@ class DatabaseFunctions:
         # target_data is an iterable
 
         if column_name == '*':
-            self.database.execute('DELETE FROM books')
+            self.database.execute(
+                "DELETE FROM books WHERE NOT Addition = 'manual'")
         else:
-            sql_command = f'DELETE FROM books WHERE {column_name} = ?'
+            sql_command = f"DELETE FROM books WHERE {column_name} = ?"
             for i in target_data:
                 self.database.execute(sql_command, (i,))
 

lector/library.py

@@ -41,7 +41,8 @@ class Library:
             books = database.DatabaseFunctions(
                 self.parent.database_path).fetch_data(
                     ('Title', 'Author', 'Year', 'DateAdded', 'Path',
-                     'Position', 'ISBN', 'Tags', 'Hash', 'LastAccessed'),
+                     'Position', 'ISBN', 'Tags', 'Hash', 'LastAccessed',
+                     'Addition'),
                     'books',
                     {'Title': ''},
                     'LIKE')
@@ -64,7 +65,7 @@ class Library:
 
                 books.append([
                     i[1]['title'], i[1]['author'], i[1]['year'], current_qdatetime,
-                    i[1]['path'], None, i[1]['isbn'], _tags, i[0], None])
+                    i[1]['path'], None, i[1]['isbn'], _tags, i[0], None, i[1]['addition_mode']])
 
         else:
             return
@@ -76,6 +77,8 @@ class Library:
             author = i[1]
             year = i[2]
             path = i[4]
+            addition_mode = i[10]
+
             last_accessed = i[9]
             if last_accessed and not isinstance(last_accessed, QtCore.QDateTime):
                 last_accessed = pickle.loads(last_accessed)
@@ -121,6 +124,7 @@ class Library:
                 'tags': tags,
                 'hash': i[8],
                 'last_accessed': last_accessed,
+                'addition_mode': addition_mode,
                 'file_exists': file_exists}
 
             author_string = self._translate('Library', 'Author')
@@ -240,12 +244,21 @@ class Library:
                 {'Path': ''},
                 'LIKE')
 
-        if not db_library_directories:  # Empty database / table
+        if db_library_directories:  # Empty database / table
-            return
-
             library_directories = {
                 i[0]: (i[1], i[2]) for i in db_library_directories}
 
+        else:
+            db_library_directories = database.DatabaseFunctions(
+                self.parent.database_path).fetch_data(
+                    ('Path',),
+                    'books',  # This checks the directories table NOT the book one
+                    {'Path': ''},
+                    'LIKE')
+
+            library_directories = {
+                i[0]: (None, None) for i in db_library_directories}
+
         def get_tags(all_metadata):
             path = os.path.dirname(all_metadata['path'])
             path_ref = pathlib.Path(path)
@@ -264,6 +277,8 @@ class Library:
 
                     return directory_name, directory_tags
 
+            # A file is assigned a 'manually added' tag in case it isn't
+            # in any designated library directory
             added_string = self._translate('Library', 'manually added')
             return added_string.lower(), None
 
@@ -281,23 +296,22 @@ class Library:
         # All files in unselected directories will have to be removed
         # from both of the models
         # They will also have to be deleted from the library
-        valid_paths = set(valid_paths)
+        # valid_paths = set(valid_paths)
-
-        # Get all paths
-        all_paths = set()
-        for i in range(self.view_model.rowCount()):
-            item = self.view_model.item(i, 0)
-            item_metadata = item.data(QtCore.Qt.UserRole + 3)
-            book_path = item_metadata['path']
-            all_paths.add(book_path)
-
-        invalid_paths = all_paths - valid_paths
 
+        invalid_paths = []
         deletable_persistent_indexes = []
+
         for i in range(self.view_model.rowCount()):
             item = self.view_model.item(i)
-            path = item.data(QtCore.Qt.UserRole + 3)['path']
+
-            if path in invalid_paths:
+            item_metadata = item.data(QtCore.Qt.UserRole + 3)
+            book_path = item_metadata['path']
+            addition_mode = item_metadata['addition_mode']
+
+            if (book_path not in valid_paths and
+                    (addition_mode != 'manual' or addition_mode is None)):
+
+                invalid_paths.append(book_path)
                 deletable_persistent_indexes.append(
                     QtCore.QPersistentModelIndex(item.index()))
 

lector/settingsdialog.py

@@ -174,9 +174,6 @@ class SettingsUI(QtWidgets.QDialog, settingswindow.Ui_Dialog):
             self.treeView.hideColumn(i)
 
     def start_library_scan(self):
-        # TODO
-        # return in case the treeView is not edited
-
         self.hide()
 
         data_pairs = []
@@ -195,15 +192,13 @@ class SettingsUI(QtWidgets.QDialog, settingswindow.Ui_Dialog):
             except AttributeError:
                 pass
 
-            self.parent.lib_ref.view_model.clear()
-            self.parent.lib_ref.table_rows = []
-
-            # TODO
-            # Change this to no longer include files added manually
-
             database.DatabaseFunctions(
                 self.database_path).delete_from_database('*', '*')
 
+            self.parent.lib_ref.generate_model('build')
+            self.parent.lib_ref.generate_proxymodels()
+            self.parent.generate_library_filter_menu()
+
             return
 
         # Update the main window library filter menu
@@ -237,7 +232,7 @@ class SettingsUI(QtWidgets.QDialog, settingswindow.Ui_Dialog):
 
         # We now create a new thread to put those files into the database
         self.thread = BackGroundBookAddition(
-            self.thread.valid_files, self.database_path, True, self.parent)
+            self.thread.valid_files, self.database_path, 'automatic', self.parent)
         self.thread.finished.connect(self.parent.move_on)
         self.thread.start()
 

lector/sorter.py

@@ -87,7 +87,8 @@ class BookSorter:
         self.file_list = [i for i in file_list if os.path.exists(i)]
         self.statistics = [0, (len(file_list))]
         self.hashes_and_paths = {}
-        self.mode = mode
+        self.work_mode = mode[0]
+        self.addition_mode = mode[1]
         self.database_path = database_path
         self.auto_tags = auto_tags
         self.temp_dir = temp_dir
@@ -98,7 +99,7 @@ class BookSorter:
         self.queue = Manager().Queue()
         self.processed_books = []
 
-        if self.mode == 'addition':
+        if self.work_mode == 'addition':
             progress_object_generator()
 
     def database_hashes(self):
@@ -153,7 +154,7 @@ class BookSorter:
 
         # Do not allow addition in case the file
         # is already in the database and it remains at its original path
-        if self.mode == 'addition' and file_md5 in self.hashes_and_paths:
+        if self.work_mode == 'addition' and file_md5 in self.hashes_and_paths:
             if (self.hashes_and_paths[file_md5] == filename
                     or os.path.exists(self.hashes_and_paths[file_md5])):
 
@@ -181,7 +182,7 @@ class BookSorter:
                 'path': filename}
 
             # Different modes require different values
-            if self.mode == 'addition':
+            if self.work_mode == 'addition':
                 # Reduce the size of the incoming image
                 # if one is found
                 title = book_ref.get_title()
@@ -205,8 +206,9 @@ class BookSorter:
                     cover_image = None
 
                 this_book[file_md5]['cover_image'] = cover_image
+                this_book[file_md5]['addition_mode'] = self.addition_mode
 
-            if self.mode == 'reading':
+            if self.work_mode == 'reading':
                 all_content = book_ref.get_contents()
 
                 # get_contents() returns a tuple. Index 1 is a collection of

lector/threaded.py

@@ -44,29 +44,31 @@ class BackGroundTabUpdate(QtCore.QThread):
 
 
 class BackGroundBookAddition(QtCore.QThread):
-    def __init__(self, file_list, database_path, prune_required, parent=None):
+    def __init__(self, file_list, database_path, addition_mode, parent=None):
         super(BackGroundBookAddition, self).__init__(parent)
         self.file_list = file_list
         self.parent = parent
         self.database_path = database_path
-        self.prune_required = prune_required
+        self.addition_mode = addition_mode
+
+        self.prune_required = True
+        if self.addition_mode == 'manual':
+            self.prune_required = False
 
     def run(self):
         books = sorter.BookSorter(
             self.file_list,
-            'addition',
+            ('addition', self.addition_mode),
             self.database_path,
             self.parent.settings['auto_tags'],
             self.parent.temp_dir.path())
 
         parsed_books = books.initiate_threads()
-        if not parsed_books:
-            return
-
         self.parent.lib_ref.generate_model('addition', parsed_books, False)
+        database.DatabaseFunctions(self.database_path).add_to_database(parsed_books)
+
         if self.prune_required:
             self.parent.lib_ref.prune_models(self.file_list)
-        database.DatabaseFunctions(self.database_path).add_to_database(parsed_books)
 
 
 class BackGroundBookDeletion(QtCore.QThread):