diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt
index caf5f64f..fb442729 100644
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt
@@ -90,7 +90,7 @@ object CategoryManga {
         return transaction {
             CategoryMangaTable.innerJoin(MangaTable)
                 .slice(selectedColumns)
-                .select { CategoryMangaTable.category eq categoryId }
+                .select { (MangaTable.inLibrary eq true) and (CategoryMangaTable.category eq categoryId) }
                 .map(transform)
         }
     }
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt
index 6956225a..fb4ca318 100644
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Library.kt
@@ -7,7 +7,6 @@ package suwayomi.tachidesk.manga.impl
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
@@ -24,17 +23,20 @@ object Library {
         if (!manga.inLibrary) {
             transaction {
                 val defaultCategories = CategoryTable.select { CategoryTable.isDefault eq true }.toList()
+                val existingCategories = CategoryMangaTable.select { CategoryMangaTable.manga eq mangaId }.toList()
 
                 MangaTable.update({ MangaTable.id eq manga.id }) {
                     it[inLibrary] = true
                     it[inLibraryAt] = Instant.now().epochSecond
-                    it[defaultCategory] = defaultCategories.isEmpty()
+                    it[defaultCategory] = defaultCategories.isEmpty() && existingCategories.isEmpty()
                 }
 
-                defaultCategories.forEach { category ->
-                    CategoryMangaTable.insert {
-                        it[CategoryMangaTable.category] = category[CategoryTable.id].value
-                        it[CategoryMangaTable.manga] = mangaId
+                if (existingCategories.isEmpty()) {
+                    defaultCategories.forEach { category ->
+                        CategoryMangaTable.insert {
+                            it[CategoryMangaTable.category] = category[CategoryTable.id].value
+                            it[CategoryMangaTable.manga] = mangaId
+                        }
                     }
                 }
             }
@@ -47,9 +49,7 @@ object Library {
             transaction {
                 MangaTable.update({ MangaTable.id eq manga.id }) {
                     it[inLibrary] = false
-                    it[defaultCategory] = true
                 }
-                CategoryMangaTable.deleteWhere { CategoryMangaTable.manga eq mangaId }
             }
         }
     }