From 1d7a60b630a8f4401ceaad641d5b49b2230b2a86 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Thu, 12 Jun 2025 17:39:19 +0200 Subject: [PATCH] Automatically truncate required varchar columns (#1423) --- .../suwayomi/tachidesk/manga/impl/Manga.kt | 12 +----- .../manga/model/table/ChapterTable.kt | 5 ++- .../tachidesk/manga/model/table/MangaTable.kt | 11 ++--- .../manga/model/table/TrackRecordTable.kt | 3 +- .../manga/model/table/TrackSearchTable.kt | 43 ++++++++++--------- .../table/columns/TruncatingVarCharColumn.kt | 36 ++++++++++++++++ 6 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/columns/TruncatingVarCharColumn.kt diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index 5fddac09..0daa8ce1 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -58,16 +58,6 @@ import java.time.Instant private val logger = KotlinLogging.logger { } object Manga { - private fun truncate( - text: String?, - maxLength: Int, - ): String? = - if (text?.length ?: 0 > maxLength) { - text?.take(maxLength - 3) + "..." - } else { - text - } - suspend fun getManga( mangaId: Int, onlineFetch: Boolean = false, @@ -148,7 +138,7 @@ object Manga { it[MangaTable.artist] = sManga.artist ?: mangaEntry[MangaTable.artist] it[MangaTable.author] = sManga.author ?: mangaEntry[MangaTable.author] - it[MangaTable.description] = sManga.description?.let { truncate(it, 4096) } + it[MangaTable.description] = sManga.description ?: mangaEntry[MangaTable.description] it[MangaTable.genre] = sManga.genre ?: mangaEntry[MangaTable.genre] it[MangaTable.status] = sManga.status diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt index 6d7f8f65..7de68f74 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt @@ -14,13 +14,14 @@ import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.transactions.transaction import suwayomi.tachidesk.manga.impl.Chapter.getChapterMetaMap import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass +import suwayomi.tachidesk.manga.model.table.columns.truncatingVarchar object ChapterTable : IntIdTable() { val url = varchar("url", 2048) - val name = varchar("name", 512) + val name = truncatingVarchar("name", 512) val date_upload = long("date_upload").default(0) val chapter_number = float("chapter_number").default(-1f) - val scanlator = varchar("scanlator", 128).nullable() + val scanlator = truncatingVarchar("scanlator", 128).nullable() val isRead = bool("read").default(false) val isBookmarked = bool("bookmark").default(false) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt index aede63da..e0924c6f 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt @@ -16,16 +16,17 @@ import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass import suwayomi.tachidesk.manga.model.dataclass.toGenreList import suwayomi.tachidesk.manga.model.table.MangaStatus.Companion +import suwayomi.tachidesk.manga.model.table.columns.truncatingVarchar object MangaTable : IntIdTable() { val url = varchar("url", 2048) - val title = varchar("title", 512) + val title = truncatingVarchar("title", 512) val initialized = bool("initialized").default(false) - val artist = varchar("artist", Integer.MAX_VALUE).nullable() - val author = varchar("author", Integer.MAX_VALUE).nullable() - val description = varchar("description", Integer.MAX_VALUE).nullable() - val genre = varchar("genre", Integer.MAX_VALUE).nullable() + val artist = truncatingVarchar("artist", Integer.MAX_VALUE).nullable() + val author = truncatingVarchar("author", Integer.MAX_VALUE).nullable() + val description = truncatingVarchar("description", Integer.MAX_VALUE).nullable() + val genre = truncatingVarchar("genre", Integer.MAX_VALUE).nullable() val status = integer("status").default(SManga.UNKNOWN) val thumbnail_url = varchar("thumbnail_url", 2048).nullable() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/TrackRecordTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/TrackRecordTable.kt index 44903df7..ced2b5d0 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/TrackRecordTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/TrackRecordTable.kt @@ -9,13 +9,14 @@ package suwayomi.tachidesk.manga.model.table import org.jetbrains.exposed.dao.id.IntIdTable import org.jetbrains.exposed.sql.ReferenceOption +import suwayomi.tachidesk.manga.model.table.columns.truncatingVarchar object TrackRecordTable : IntIdTable() { val mangaId = reference("manga_id", MangaTable, ReferenceOption.CASCADE) val trackerId = integer("sync_id") val remoteId = long("remote_id") val libraryId = long("library_id").nullable() - val title = varchar("title", 512) + val title = truncatingVarchar("title", 512) val lastChapterRead = double("last_chapter_read") val totalChapters = integer("total_chapters") val status = integer("status") diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/TrackSearchTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/TrackSearchTable.kt index 660a9c86..81c68bff 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/TrackSearchTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/TrackSearchTable.kt @@ -16,18 +16,19 @@ import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.statements.BatchUpdateStatement import org.jetbrains.exposed.sql.transactions.transaction import suwayomi.tachidesk.manga.impl.track.tracker.model.TrackSearch +import suwayomi.tachidesk.manga.model.table.columns.truncatingVarchar object TrackSearchTable : IntIdTable() { val trackerId = integer("tracker_id") val remoteId = long("remote_id") - val title = varchar("title", 512) + val title = truncatingVarchar("title", 512) val totalChapters = integer("total_chapters") - val trackingUrl = varchar("tracking_url", 512) - val coverUrl = varchar("cover_url", 512) - val summary = varchar("summary", 4096) - val publishingStatus = varchar("publishing_status", 512) - val publishingType = varchar("publishing_type", 512) - val startDate = varchar("start_date", 128) + val trackingUrl = truncatingVarchar("tracking_url", 512) + val coverUrl = truncatingVarchar("cover_url", 512) + val summary = truncatingVarchar("summary", 4096) + val publishingStatus = truncatingVarchar("publishing_status", 512) + val publishingType = truncatingVarchar("publishing_type", 512) + val startDate = truncatingVarchar("start_date", 128) } fun List.insertAll(): List { @@ -63,14 +64,14 @@ fun List.insertAll(): List { toUpdate.forEach { (id, trackSearch) -> id ?: return@forEach addBatch(EntityID(id, TrackSearchTable)) - this[TrackSearchTable.title] = trackSearch.title.take(512) + this[TrackSearchTable.title] = trackSearch.title this[TrackSearchTable.totalChapters] = trackSearch.total_chapters - this[TrackSearchTable.trackingUrl] = trackSearch.tracking_url.take(512) - this[TrackSearchTable.coverUrl] = trackSearch.cover_url.take(512) - this[TrackSearchTable.summary] = trackSearch.summary.take(4096) - this[TrackSearchTable.publishingStatus] = trackSearch.publishing_status.take(512) - this[TrackSearchTable.publishingType] = trackSearch.publishing_type.take(512) - this[TrackSearchTable.startDate] = trackSearch.start_date.take(128) + this[TrackSearchTable.trackingUrl] = trackSearch.tracking_url + this[TrackSearchTable.coverUrl] = trackSearch.cover_url + this[TrackSearchTable.summary] = trackSearch.summary + this[TrackSearchTable.publishingStatus] = trackSearch.publishing_status + this[TrackSearchTable.publishingType] = trackSearch.publishing_type + this[TrackSearchTable.startDate] = trackSearch.start_date } execute(this@transaction) } @@ -80,14 +81,14 @@ fun List.insertAll(): List { TrackSearchTable.batchInsert(toInsert) { this[TrackSearchTable.trackerId] = it.sync_id this[TrackSearchTable.remoteId] = it.media_id - this[TrackSearchTable.title] = it.title.take(512) + this[TrackSearchTable.title] = it.title this[TrackSearchTable.totalChapters] = it.total_chapters - this[TrackSearchTable.trackingUrl] = it.tracking_url.take(512) - this[TrackSearchTable.coverUrl] = it.cover_url.take(512) - this[TrackSearchTable.summary] = it.summary.take(4096) - this[TrackSearchTable.publishingStatus] = it.publishing_status.take(512) - this[TrackSearchTable.publishingType] = it.publishing_type.take(512) - this[TrackSearchTable.startDate] = it.start_date.take(128) + this[TrackSearchTable.trackingUrl] = it.tracking_url + this[TrackSearchTable.coverUrl] = it.cover_url + this[TrackSearchTable.summary] = it.summary + this[TrackSearchTable.publishingStatus] = it.publishing_status + this[TrackSearchTable.publishingType] = it.publishing_type + this[TrackSearchTable.startDate] = it.start_date } } else { emptyList() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/columns/TruncatingVarCharColumn.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/columns/TruncatingVarCharColumn.kt new file mode 100644 index 00000000..bcadbd0f --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/columns/TruncatingVarCharColumn.kt @@ -0,0 +1,36 @@ +package suwayomi.tachidesk.manga.model.table.columns + +import io.github.oshai.kotlinlogging.KotlinLogging +import org.jetbrains.exposed.sql.Column +import org.jetbrains.exposed.sql.Table +import org.jetbrains.exposed.sql.VarCharColumnType + +class TruncatingVarCharColumn( + private val table: String, + private val name: String, + colLength: Int = 255, + collate: String? = null, +) : VarCharColumnType(colLength, collate) { + private val logger = KotlinLogging.logger { } + + override fun sqlType(): String = "varchar($colLength)" + + override fun notNullValueToDB(value: String): Any { + if (value.length > colLength) { + logger.warn { "Value of column \"$table::$name\" exceeds length (${value.length} > $colLength)" } + return value.take(colLength - 3) + "..." + } + + return value + } + + override fun validateValueBeforeUpdate(value: String?) { + // not necessary, value gets truncated before inserting it into the database + } +} + +fun Table.truncatingVarchar( + name: String, + length: Int, + collate: String? = null, +): Column = registerColumn(name, TruncatingVarCharColumn(this.tableName, name, length, collate))