/*! * Copyright (c) 2024 PLANKA Software GmbH * Licensed under the Fair Use License: https://github.com/plankanban/planka/blob/master/LICENSE.md */ /* eslint-disable no-await-in-loop */ /* eslint-disable no-console */ /* eslint-disable no-continue */ /* eslint-disable no-restricted-syntax */ const { getEncoding } = require('istextorbinary'); const mime = require('mime'); const uuid = require('uuid'); const sharp = require('sharp'); const initKnex = require('knex'); const sails = require('sails'); const rc = require('sails/accessible/rc'); const _ = require('lodash'); const knexfile = require('./knexfile'); const { MAX_SIZE_IN_BYTES_TO_GET_ENCODING, POSITION_GAP } = require('../constants'); const PrevActionTypes = { COMMENT_CARD: 'commentCard', }; const PROJECT_BACKGROUND_IMAGES_PATH_SEGMENT = 'public/project-background-images'; const readStreamToBuffer = (readStream) => new Promise((resolve, reject) => { const chunks = []; readStream.on('data', (chunk) => { chunks.push(chunk); }); readStream.on('end', () => { resolve(Buffer.concat(chunks)); }); readStream.on('error', (error) => { reject(error); }); }); const loadSails = () => new Promise((resolve) => { sails.load( { ...rc('sails'), log: { level: 'error', }, }, resolve, ); }); const knex = initKnex(knexfile); const upgradeDatabase = async () => { await knex.transaction(async (trx) => { await trx.raw(` CREATE SCHEMA v1; ALTER SEQUENCE next_id_seq SET SCHEMA v1; ALTER FUNCTION next_id(OUT id BIGINT) SET SCHEMA v1; ALTER TABLE migration SET SCHEMA v1; ALTER TABLE migration_lock SET SCHEMA v1; ALTER TABLE archive SET SCHEMA v1; ALTER TABLE user_account SET SCHEMA v1; ALTER TABLE identity_provider_user SET SCHEMA v1; ALTER TABLE session SET SCHEMA v1; ALTER TABLE project SET SCHEMA v1; ALTER TABLE project_manager SET SCHEMA v1; ALTER TABLE board SET SCHEMA v1; ALTER TABLE board_membership SET SCHEMA v1; ALTER TABLE label SET SCHEMA v1; ALTER TABLE list SET SCHEMA v1; ALTER TABLE card SET SCHEMA v1; ALTER TABLE card_subscription SET SCHEMA v1; ALTER TABLE card_membership SET SCHEMA v1; ALTER TABLE card_label SET SCHEMA v1; ALTER TABLE task SET SCHEMA v1; ALTER TABLE attachment SET SCHEMA v1; ALTER TABLE action SET SCHEMA v1; ALTER TABLE notification SET SCHEMA v1; `); await trx.migrate.up({ name: '20250228000022_version_2.js', }); const users = await trx('user_account').withSchema('v1').whereNull('deleted_at'); const userIds = users.map(({ id }) => id); const userIdsSet = new Set(userIds); const whereInUserIds = ['0', ...userIds]; if (users.length > 0) { await knex .batchInsert( 'user_account', users.map((user) => ({ ..._.pick(user, [ 'id', 'email', 'password', 'name', 'username', 'phone', 'organization', 'language', 'subscribe_to_own_cards', 'created_at', 'updated_at', 'password_changed_at', ]), role: user.is_admin ? User.Roles.ADMIN : User.Roles.BOARD_USER, avatar: user.avatar && { ...user.avatar, sizeInBytes: 0, }, subscribe_to_card_when_commenting: true, turn_off_recent_card_highlighting: false, enable_favorites_by_default: false, default_editor_mode: User.EditorModes.WYSIWYG, default_home_view: User.HomeViews.GROUPED_PROJECTS, default_projects_order: User.ProjectOrders.BY_DEFAULT, is_sso_user: user.is_sso, is_deactivated: false, })), ) .transacting(trx); const identityProviderUsers = await trx('identity_provider_user') .withSchema('v1') .whereIn('user_id', whereInUserIds); if (identityProviderUsers.length > 0) { await knex.batchInsert('identity_provider_user', identityProviderUsers).transacting(trx); } const sessions = await trx('session').withSchema('v1').whereIn('user_id', whereInUserIds); if (sessions.length > 0) { await knex.batchInsert('session', sessions).transacting(trx); } } const projects = await trx('project').withSchema('v1'); const whereInProjectIds = ['0', ...projects.map(({ id }) => id)]; if (projects.length > 0) { const projectsWithBackgroundImage = projects.filter((project) => project.background_image); const backgroundImageIdByProjectId = {}; if (projectsWithBackgroundImage.length > 0) { const createdAt = new Date().toISOString(); const backgroundImages = await knex .batchInsert( 'background_image', projectsWithBackgroundImage.map((project) => ({ ...project.background_image, project_id: project.id, size_in_bytes: 0, created_at: createdAt, })), ) .returning(['id', 'project_id']) .transacting(trx); backgroundImages.forEach((backgroundImage) => { backgroundImageIdByProjectId[backgroundImage.project_id] = backgroundImage.id; }); } await knex .batchInsert( 'project', projects.map((project) => { const data = { ..._.pick(project, ['id', 'name', 'created_at', 'updated_at']), is_hidden: false, }; if (project.background) { data.background_type = project.background.type; switch (project.background.type) { case Project.BackgroundTypes.GRADIENT: data.background_gradient = project.background.name; break; case Project.BackgroundTypes.IMAGE: data.background_image_id = backgroundImageIdByProjectId[project.id]; break; default: } } return data; }), ) .transacting(trx); } const projectManagers = await trx('project_manager') .withSchema('v1') .whereIn('project_id', whereInProjectIds); if (projectManagers.length > 0) { await knex.batchInsert('project_manager', projectManagers).transacting(trx); } const boards = await trx('board').withSchema('v1').whereIn('project_id', whereInProjectIds); const projectIdByBoardId = boards.reduce( (result, board) => ({ ...result, [board.id]: board.project_id, }), {}, ); const whereInBoardIds = ['0', ...Object.keys(projectIdByBoardId)]; if (boards.length > 0) { await knex .batchInsert( 'board', boards.map((board) => ({ ..._.pick(board, ['id', 'project_id', 'position', 'name', 'created_at', 'updated_at']), default_view: Board.Views.KANBAN, default_card_type: Card.Types.PROJECT, limit_card_types_to_default_one: false, always_display_card_creator: false, })), ) .transacting(trx); const createdAt = new Date().toISOString(); await knex .batchInsert( 'list', boards.flatMap((board) => [List.Types.ARCHIVE, List.Types.TRASH].map((type) => ({ type, board_id: board.id, created_at: createdAt, })), ), ) .transacting(trx); } const boardMemberships = await trx('board_membership') .withSchema('v1') .whereIn('board_id', whereInBoardIds); if (boardMemberships.length > 0) { await knex .batchInsert( 'board_membership', boardMemberships.map((boardMembership) => ({ ..._.pick(boardMembership, [ 'id', 'board_id', 'user_id', 'role', 'can_comment', 'created_at', 'updated_at', ]), project_id: projectIdByBoardId[boardMembership.board_id], })), ) .transacting(trx); } const labels = await trx('label').withSchema('v1').whereIn('board_id', whereInBoardIds); if (labels.length > 0) { await knex.batchInsert('label', labels).transacting(trx); } const lists = await trx('list').withSchema('v1').whereIn('board_id', whereInBoardIds); const whereInListIds = ['0', ...lists.map(({ id }) => id)]; if (lists.length > 0) { await knex .batchInsert( 'list', lists.map((list) => ({ ..._.pick(list, [ 'id', 'board_id', 'position', 'name', 'color', 'created_at', 'updated_at', ]), type: List.Types.ACTIVE, })), ) .transacting(trx); } const cards = await trx('card') .withSchema('v1') .whereIn('board_id', whereInBoardIds) .whereIn('list_id', whereInListIds); const cardById = _.keyBy(cards, 'id'); const whereInCardIds = ['0', ...Object.keys(cardById)]; if (cards.length > 0) { await knex .batchInsert( 'card', cards.map((card) => ({ ..._.pick(card, [ 'id', 'board_id', 'list_id', 'cover_attachment_id', 'position', 'name', 'description', 'due_date', 'stopwatch', 'created_at', 'updated_at', ]), creator_user_id: userIdsSet.has(card.creator_user_id) ? card.creator_user_id : null, type: Card.Types.PROJECT, list_changed_at: card.created_at, })), ) .transacting(trx); } const cardSubscriptions = await trx('card_subscription') .withSchema('v1') .whereIn('card_id', whereInCardIds); if (cardSubscriptions.length > 0) { await knex.batchInsert('card_subscription', cardSubscriptions).transacting(trx); } const cardMemberships = await trx('card_membership') .withSchema('v1') .whereIn('card_id', whereInCardIds); if (cardMemberships.length > 0) { await knex.batchInsert('card_membership', cardMemberships).transacting(trx); } const cardLabels = await trx('card_label').withSchema('v1').whereIn('card_id', whereInCardIds); if (cardLabels.length > 0) { await knex.batchInsert('card_label', cardLabels).transacting(trx); } const tasks = await trx('task').withSchema('v1').whereIn('card_id', whereInCardIds); const tasksByCardId = _.groupBy(tasks, 'card_id'); const taskCardIds = Object.keys(tasksByCardId); if (taskCardIds.length > 0) { const createdAt = new Date().toISOString(); const taskLists = await knex .batchInsert( 'task_list', taskCardIds.map((cardId) => ({ card_id: cardId, position: POSITION_GAP, name: 'Task List', show_on_front_of_card: true, created_at: createdAt, })), ) .returning(['id', 'card_id']) .transacting(trx); await knex .batchInsert( 'task', taskLists.flatMap((taskList) => tasksByCardId[taskList.card_id].map((task) => ({ ..._.pick(task, [ 'id', 'position', 'name', 'is_completed', 'created_at', 'updated_at', ]), task_list_id: taskList.id, })), ), ) .transacting(trx); } const attachments = await trx('attachment').withSchema('v1').whereIn('card_id', whereInCardIds); if (attachments.length > 0) { await knex .batchInsert( 'attachment', attachments.map((attachment) => ({ ..._.pick(attachment, ['id', 'card_id', 'name', 'created_at', 'updated_at']), creator_user_id: userIdsSet.has(attachment.creator_user_id) ? attachment.creator_user_id : null, type: Attachment.Types.FILE, data: { fileReferenceId: attachment.dirname, filename: attachment.filename, mimeType: mime.getType(attachment.filename), sizeInBytes: 0, encoding: null, image: attachment.image, }, })), ) .transacting(trx); } const actions = await trx('action').withSchema('v1').whereIn('card_id', whereInCardIds); const actionById = _.keyBy(actions, 'id'); const whereInActionIds = ['0', ...Object.keys(actionById)]; const commentActions = []; const otherActions = []; actions.forEach((action) => { if (action.type === PrevActionTypes.COMMENT_CARD) { commentActions.push(action); } else { otherActions.push(action); } }); if (commentActions.length > 0) { await knex .batchInsert( 'comment', commentActions.map((action) => ({ ..._.pick(action, ['id', 'card_id', 'created_at', 'updated_at']), user_id: userIdsSet.has(action.user_id) ? action.user_id : null, text: action.data.text, })), ) .transacting(trx); } if (otherActions.length > 0) { await knex .batchInsert( 'action', otherActions.map((action) => { const data = { ..._.pick(action, ['id', 'card_id', 'type', 'created_at', 'updated_at']), user_id: userIdsSet.has(action.user_id) ? action.user_id : null, }; switch (action.type) { case Action.Types.CREATE_CARD: data.data = { list: { ...action.data.list, type: List.Types.ACTIVE, }, }; break; case Action.Types.MOVE_CARD: data.data = { fromList: { ...action.data.fromList, type: List.Types.ACTIVE, }, toList: { ...action.data.toList, type: List.Types.ACTIVE, }, }; break; default: } return data; }), ) .transacting(trx); } const notifications = await trx('notification') .withSchema('v1') .whereIn('user_id', whereInUserIds) .whereIn('action_id', whereInActionIds) .whereIn('card_id', whereInCardIds); if (notifications.length > 0) { await knex .batchInsert( 'notification', notifications.map((notification) => { const card = cardById[notification.card_id]; const action = actionById[notification.action_id]; const data = { ..._.pick(notification, [ 'id', 'user_id', 'card_id', 'is_read', 'created_at', 'updated_at', ]), creator_user_id: userIdsSet.has(notification.creator_user_id) ? notification.creator_user_id : null, board_id: card.board_id, type: action.type, }; if (action.type === PrevActionTypes.COMMENT_CARD) { Object.assign(data, { comment_id: action.id, data: { card: _.pick(card, ['name']), text: action.data.text, }, }); } else { Object.assign(data, { action_id: action.id, data: { fromList: { ...action.data.fromList, type: List.Types.ACTIVE, }, toList: { ...action.data.toList, type: List.Types.ACTIVE, }, card: _.pick(card, ['name']), }, }); } return data; }), ) .transacting(trx); } await trx.schema.dropSchema('v1', true); }); }; const upgradeUserAvatars = async () => { const fileManager = sails.hooks['file-manager'].getInstance(); const dirnames = await fileManager.listDir(sails.config.custom.userAvatarsPathSegment); const users = await knex('user_account').whereNotNull('avatar'); if (dirnames) { const userByDirname = _.keyBy(users, 'avatar.dirname'); for (const dirname of dirnames) { const user = userByDirname[dirname]; const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${dirname}`; if (user) { const sizeInBytes = await fileManager.getSizeInBytes( `${dirPathSegment}/original.${user.avatar.extension}`, ); await knex('user_account') .update({ avatar: knex.raw("?? || jsonb_build_object('sizeInBytes', ?::bigint)", [ 'avatar', sizeInBytes, ]), }) .where('id', user.id); } else { await fileManager.deleteDir(dirPathSegment); } } } for (const { avatar } of users) { const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${avatar.dirname}`; const isExists = await fileManager.isExists(`${dirPathSegment}/cover-180.${avatar.extension}`); if (isExists) { continue; } await fileManager.delete(`${dirPathSegment}/square-100.${avatar.extension}`); let readStream; try { readStream = await fileManager.read(`${dirPathSegment}/original.${avatar.extension}`); } catch (error) { continue; } const originalBuffer = await readStreamToBuffer(readStream); const image = sharp(originalBuffer, { animated: true, }); const cover180Buffer = await image .resize(180, 180, { withoutEnlargement: true, }) .png({ quality: 75, force: false, }) .toBuffer(); await fileManager.save( `${dirPathSegment}/cover-180.${avatar.extension}`, cover180Buffer, mime.getType(avatar.extension), ); } }; const upgradeBackgroundImages = async () => { const fileManager = sails.hooks['file-manager'].getInstance(); await fileManager.renameDir( PROJECT_BACKGROUND_IMAGES_PATH_SEGMENT, sails.config.custom.backgroundImagesPathSegment, ); const dirnames = await fileManager.listDir(sails.config.custom.backgroundImagesPathSegment); const backgroundImages = await knex('background_image'); if (dirnames) { const backgroundImageByDirname = _.keyBy(backgroundImages, 'dirname'); for (const dirname of dirnames) { const backgroundImage = backgroundImageByDirname[dirname]; const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${dirname}`; if (backgroundImage) { const sizeInBytes = await fileManager.getSizeInBytes( `${dirPathSegment}/original.${backgroundImage.extension}`, ); await knex('background_image') .update({ size_in_bytes: sizeInBytes, }) .where('id', backgroundImage.id); } else { await fileManager.deleteDir(dirPathSegment); } } } for (const backgroundImage of backgroundImages) { const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${backgroundImage.dirname}`; const isExists = await fileManager.isExists( `${dirPathSegment}/outside-360.${backgroundImage.extension}`, ); if (isExists) { continue; } await fileManager.delete(`${dirPathSegment}/cover-336.${backgroundImage.extension}`); let readStream; try { readStream = await fileManager.read( `${dirPathSegment}/original.${backgroundImage.extension}`, ); } catch (error) { continue; } const originalBuffer = await readStreamToBuffer(readStream); const image = sharp(originalBuffer, { animated: true, }); const outside360Buffer = await image .resize(360, 360, { fit: 'outside', withoutEnlargement: true, }) .png({ quality: 75, force: false, }) .toBuffer(); await fileManager.save( `${dirPathSegment}/outside-360.${backgroundImage.extension}`, outside360Buffer, mime.getType(backgroundImage.extension), ); } }; const upgradeFileAttachments = async () => { const fileManager = sails.hooks['file-manager'].getInstance(); const dirnames = await fileManager.listDir(sails.config.custom.attachmentsPathSegment); const attachments = await knex('attachment').where('type', Attachment.Types.FILE); const fileReferenceIds = []; if (dirnames) { const attachmentByDirname = _.keyBy(attachments, 'data.fileReferenceId'); for (const dirname of dirnames) { const attachment = attachmentByDirname[dirname]; const dirPathSegment = `${sails.config.custom.attachmentsPathSegment}/${dirname}`; if (attachment) { if (uuid.validate(dirname)) { const fileReferenceId = await knex.transaction(async (trx) => { const [{ id }] = await trx('file_reference').insert( { total: 1, created_at: new Date().toISOString(), }, 'id', ); const sizeInBytes = await fileManager.getSizeInBytes( `${dirPathSegment}/${attachment.data.filename}`, ); let encoding = null; if (sizeInBytes && sizeInBytes <= MAX_SIZE_IN_BYTES_TO_GET_ENCODING) { const readStream = await fileManager.read( `${dirPathSegment}/${attachment.data.filename}`, ); const buffer = await readStreamToBuffer(readStream); encoding = getEncoding(buffer); } await trx('attachment') .update({ data: trx.raw( "?? || jsonb_build_object('fileReferenceId', ?::text, 'sizeInBytes', ?::bigint, 'encoding', ?::text)", ['data', id, sizeInBytes, encoding], ), }) .where('id', attachment.id); await fileManager.renameDir( `${dirPathSegment}`, `${sails.config.custom.attachmentsPathSegment}/${id}`, ); return id; }); fileReferenceIds.push(fileReferenceId); } else { fileReferenceIds.push(dirname); } } else { await fileManager.deleteDir(dirPathSegment); } } } if (fileReferenceIds.length > 0) { await knex('attachment') .update({ data: knex.raw('?? || \'{"fileReferenceId":null}\'', 'data'), }) .where('type', Attachment.Types.FILE) .whereRaw(`??->>'fileReferenceId' NOT IN (${fileReferenceIds.map(() => '?').join(', ')})`, [ 'data', ...fileReferenceIds, ]); } const imageAttachments = await knex('attachment') .where('type', Attachment.Types.FILE) .whereRaw("??->>'image' IS NOT NULL", 'data'); for (const { data } of imageAttachments) { const dirPathSegment = `${sails.config.custom.attachmentsPathSegment}/${data.fileReferenceId}`; const thumbnailsPathSegment = `${dirPathSegment}/thumbnails`; const isExists = await fileManager.isExists( `${thumbnailsPathSegment}/outside-720.${data.image.thumbnailsExtension}`, ); if (isExists) { continue; } await fileManager.deleteDir(thumbnailsPathSegment); let readStream; try { readStream = await fileManager.read(`${dirPathSegment}/${data.filename}`); } catch (error) { continue; } const buffer = await readStreamToBuffer(readStream); const image = sharp(buffer, { animated: true, }); const outside360Buffer = await image .resize(360, 360, { fit: 'outside', withoutEnlargement: true, }) .png({ quality: 75, force: false, }) .toBuffer(); await fileManager.save( `${thumbnailsPathSegment}/outside-360.${data.image.thumbnailsExtension}`, outside360Buffer, data.mimeType, ); const outside720Buffer = await image .resize(720, 720, { fit: 'outside', withoutEnlargement: true, }) .png({ quality: 75, force: false, }) .toBuffer(); await fileManager.save( `${thumbnailsPathSegment}/outside-720.${data.image.thumbnailsExtension}`, outside720Buffer, data.mimeType, ); } }; (async () => { try { let migrations; try { migrations = await knex('migration').orderBy('id'); } catch (error) { throw new Error('Nothing to upgrade'); } const migrationNames = migrations.map(({ name }) => name); const isV1 = migrationNames[0] === '20180721020022_create_next_id_function.js'; const isLatestV1 = migrationNames.at(-1) === '20250131202710_add_list_color.js'; if (isV1 && !isLatestV1) { throw new Error('Update to latest v1 first'); } await loadSails(); if (isV1) { console.log('Upgrading database...'); await upgradeDatabase(); } console.log('Upgrading files...'); await upgradeUserAvatars(); await upgradeBackgroundImages(); await upgradeFileAttachments(); } catch (error) { console.error(error); process.exitCode = 1; } finally { knex.destroy(); process.exit(); } })();