From 052edc9fb1ad751c48f710fbe07b4248b2a6c395 Mon Sep 17 00:00:00 2001 From: Maksim Eltyshev Date: Sat, 31 Jan 2026 20:27:15 +0100 Subject: [PATCH] feat: Migrate file storage to unified data directory --- .dockerignore | 21 +---- Dockerfile | 6 +- charts/planka/templates/deployment.yaml | 13 +-- docker-backup.sh | 20 +---- docker-compose.yml | 10 +-- docker-restore.sh | 15 +--- server/.buildignore | 15 +--- server/.gitignore | 21 +---- .../utils/is-preloaded-favicon-exists.js | 7 +- server/config/custom.js | 10 +-- server/config/routes.js | 12 +-- server/{private/attachments => data}/.gitkeep | 0 server/db/upgrade.js | 90 +++++++++++++------ server/public/background-images/.gitkeep | 0 server/public/favicons/.gitkeep | 0 server/public/preloaded-favicons/.gitkeep | 0 server/public/user-avatars/.gitkeep | 0 17 files changed, 89 insertions(+), 151 deletions(-) rename server/{private/attachments => data}/.gitkeep (100%) delete mode 100644 server/public/background-images/.gitkeep delete mode 100644 server/public/favicons/.gitkeep delete mode 100644 server/public/preloaded-favicons/.gitkeep delete mode 100644 server/public/user-avatars/.gitkeep diff --git a/.dockerignore b/.dockerignore index da7e032f..50dcb8f5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,24 +11,9 @@ server/test server/.tmp server/.venv -server/views/* -!server/views/.gitkeep +server/views/index.ejs -server/public/* -!server/public/preloaded-favicons -!server/public/favicons -server/public/favicons/* -!server/public/favicons/.gitkeep -!server/public/user-avatars -server/public/user-avatars/* -!server/public/user-avatars/.gitkeep -!server/public/background-images -server/public/background-images/* -!server/public/background-images/.gitkeep - -server/private/* -!server/private/attachments -server/private/attachments/* -!server/private/attachments/.gitkeep +server/data/* +!server/data/.gitkeep client/dist diff --git a/Dockerfile b/Dockerfile index ebadda18..396a16b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,11 +49,7 @@ RUN python3 -m venv .venv \ && mv .env.sample .env \ && npm config set update-notifier false -VOLUME /app/public/favicons -VOLUME /app/public/user-avatars -VOLUME /app/public/background-images -VOLUME /app/private/attachments - +VOLUME /app/data EXPOSE 1337 HEALTHCHECK --interval=10s --timeout=2s --start-period=15s \ diff --git a/charts/planka/templates/deployment.yaml b/charts/planka/templates/deployment.yaml index 113a0723..cdb6aa42 100644 --- a/charts/planka/templates/deployment.yaml +++ b/charts/planka/templates/deployment.yaml @@ -54,17 +54,8 @@ spec: path: / port: http volumeMounts: - - mountPath: /app/public/favicons - subPath: favicons - name: planka - - mountPath: /app/public/user-avatars - subPath: user-avatars - name: planka - - mountPath: /app/public/background-images - subPath: background-images - name: planka - - mountPath: /app/private/attachments - subPath: attachments + - mountPath: /app/data + subPath: data name: planka {{- if .Values.securityContext.readOnlyRootFilesystem }} - mountPath: /app/logs diff --git a/docker-backup.sh b/docker-backup.sh index 6db8d9d1..c49828a9 100644 --- a/docker-backup.sh +++ b/docker-backup.sh @@ -22,28 +22,16 @@ echo -n "Exporting postgres database ... " docker exec -t "$PLANKA_DOCKER_CONTAINER_POSTGRES" pg_dumpall -c -U postgres > "$BACKUP_DATETIME-backup/postgres.sql" echo "Success!" -# Export Docker Voumes -echo -n "Exporting favicons ... " -docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/public/favicons /backup/favicons -echo "Success!" -echo -n "Exporting user-avatars ... " -docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/public/user-avatars /backup/user-avatars -echo "Success!" -echo -n "Exporting background-images ... " -docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/public/background-images /backup/background-images -echo "Success!" -echo -n "Exporting attachments ... " -docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/private/attachments /backup/attachments +# Export Docker Volume +echo -n "Exporting data volume ... " +docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/data /backup/data echo "Success!" # Create tgz echo -n "Creating final tarball $BACKUP_DATETIME-backup.tgz ... " tar -czf "$BACKUP_DATETIME-backup.tgz" \ "$BACKUP_DATETIME-backup/postgres.sql" \ - "$BACKUP_DATETIME-backup/favicons" \ - "$BACKUP_DATETIME-backup/user-avatars" \ - "$BACKUP_DATETIME-backup/background-images" \ - "$BACKUP_DATETIME-backup/attachments" + "$BACKUP_DATETIME-backup/data" echo "Success!" # Remove source files diff --git a/docker-compose.yml b/docker-compose.yml index d50340aa..e6b58213 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,10 +3,7 @@ services: image: ghcr.io/plankanban/planka:2.0.0-rc.4 restart: on-failure volumes: - - favicons:/app/public/favicons - - user-avatars:/app/public/user-avatars - - background-images:/app/public/background-images - - attachments:/app/private/attachments + - data:/app/data # Optionally override this to your user/group # user: 1000:1000 # tmpfs: @@ -132,8 +129,5 @@ services: retries: 5 volumes: - favicons: - user-avatars: - background-images: - attachments: + data: db-data: diff --git a/docker-restore.sh b/docker-restore.sh index 8dc53890..2b2cba8f 100644 --- a/docker-restore.sh +++ b/docker-restore.sh @@ -19,18 +19,9 @@ echo -n "Importing postgres database ... " cat "$PLANKA_BACKUP_ARCHIVE/postgres.sql" | docker exec -i "$PLANKA_DOCKER_CONTAINER_POSTGRES" psql -U postgres echo "Success!" -# Restore Docker Volumes -echo -n "Importing favicons ... " -docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/favicons /app/public/ -echo "Success!" -echo -n "Importing user-avatars ... " -docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/user-avatars /app/public/ -echo "Success!" -echo -n "Importing background-images ... " -docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/background-images /app/public/ -echo "Success!" -echo -n "Importing attachments ... " -docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/attachments /app/private/ +# Restore Docker Volume +echo -n "Importing data volume ... " +docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/data/. /app/public/data echo "Success!" echo -n "Cleaning up temporary files and folders ... " diff --git a/server/.buildignore b/server/.buildignore index bedcdf77..3a42c40a 100644 --- a/server/.buildignore +++ b/server/.buildignore @@ -20,17 +20,6 @@ test .tmp .venv -views/* +views/index.ejs -public/* -!public/preloaded-favicons -!public/favicons -public/favicons/* -!public/user-avatars -public/user-avatars/* -!public/background-images -public/background-images/* - -private/* -!private/attachments -private/attachments/* +data/* diff --git a/server/.gitignore b/server/.gitignore index d15cc679..26a4a62d 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -134,22 +134,7 @@ swagger.json dist logs -views/* -!views/.gitkeep +views/index.ejs -public/* -!public/preloaded-favicons -!public/favicons -public/favicons/* -!public/favicons/.gitkeep -!public/user-avatars -public/user-avatars/* -!public/user-avatars/.gitkeep -!public/background-images -public/background-images/* -!public/background-images/.gitkeep - -private/* -!private/attachments -private/attachments/* -!private/attachments/.gitkeep +data/* +!data/.gitkeep diff --git a/server/api/helpers/utils/is-preloaded-favicon-exists.js b/server/api/helpers/utils/is-preloaded-favicon-exists.js index 6babe60a..d696d484 100644 --- a/server/api/helpers/utils/is-preloaded-favicon-exists.js +++ b/server/api/helpers/utils/is-preloaded-favicon-exists.js @@ -7,12 +7,7 @@ const fs = require('fs'); const path = require('path'); const PRELOADED_FAVICON_FILENAMES = fs - .readdirSync( - path.join( - sails.config.custom.uploadsBasePath, - sails.config.custom.preloadedFaviconsPathSegment, - ), - ) + .readdirSync(path.join(sails.config.paths.public, 'preloaded-favicons')) .filter((filename) => filename.endsWith('.png')); const PRELOADED_FAVICON_HOSTNAMES_SET = new Set( diff --git a/server/config/custom.js b/server/config/custom.js index 6f8bfc59..f2fd749e 100644 --- a/server/config/custom.js +++ b/server/config/custom.js @@ -8,6 +8,7 @@ * https://sailsjs.com/config/custom */ +const path = require('path'); const { URL } = require('url'); const bytes = require('bytes'); const sails = require('sails'); @@ -48,12 +49,11 @@ module.exports.custom = { // Location to receive uploaded files in. Default (non-string value) is a Sails-specific location. uploadsTempPath: null, - uploadsBasePath: sails.config.appPath, + uploadsBasePath: path.join(sails.config.appPath, 'data'), - preloadedFaviconsPathSegment: 'public/preloaded-favicons', - faviconsPathSegment: 'public/favicons', - userAvatarsPathSegment: 'public/user-avatars', - backgroundImagesPathSegment: 'public/background-images', + faviconsPathSegment: 'protected/favicons', + userAvatarsPathSegment: 'protected/user-avatars', + backgroundImagesPathSegment: 'protected/background-images', attachmentsPathSegment: 'private/attachments', defaultAdminEmail: diff --git a/server/config/routes.js b/server/config/routes.js index 83c9ccf3..708e4c44 100644 --- a/server/config/routes.js +++ b/server/config/routes.js @@ -80,13 +80,13 @@ const serveStatic = async (prefix, getPathSegment, req, res) => { return readStream.pipe(res); }; -const publicStaticDirServer = (prefix, getPathSegment) => (req, res, next) => { +/* const publicStaticDirServer = (prefix, getPathSegment) => (req, res, next) => { if (!req.url.startsWith(prefix)) { return next(); } return serveStatic(prefix, getPathSegment, req, res); -}; +}; */ const protectedStaticDirServer = (prefix, getPathSegment) => (req, res, next) => { if (!req.url.startsWith(prefix)) { @@ -235,14 +235,6 @@ module.exports.routes = { 'PATCH /api/_internal/config': '_internal/update-config', - 'GET /preloaded-favicons/*': { - fn: publicStaticDirServer( - '/preloaded-favicons', - () => sails.config.custom.preloadedFaviconsPathSegment, - ), - skipAssets: false, - }, - 'GET /favicons/*': { fn: protectedStaticDirServer('/favicons', () => sails.config.custom.faviconsPathSegment), skipAssets: false, diff --git a/server/private/attachments/.gitkeep b/server/data/.gitkeep similarity index 100% rename from server/private/attachments/.gitkeep rename to server/data/.gitkeep diff --git a/server/db/upgrade.js b/server/db/upgrade.js index 0904c80e..52251284 100644 --- a/server/db/upgrade.js +++ b/server/db/upgrade.js @@ -24,7 +24,14 @@ const PrevActionTypes = { COMMENT_CARD: 'commentCard', }; -const PROJECT_BACKGROUND_IMAGES_PATH_SEGMENT = 'public/project-background-images'; +const PrevPathSegments = { + PROJECT_BACKGROUND_IMAGES: 'public/project-background-images', + + FAVICONS: 'public/favicons', + USER_AVATARS: 'public/user-avatars', + BACKGROUND_IMAGES: 'public/background-images', + ATTACHMENTS: 'private/attachments', +}; const readStreamToBuffer = (readStream) => new Promise((resolve, reject) => { @@ -600,7 +607,7 @@ const upgradeDatabase = async () => { const upgradeUserAvatars = async () => { const fileManager = sails.hooks['file-manager'].getInstance(); - const dirnames = await fileManager.listDir(sails.config.custom.userAvatarsPathSegment); + const dirnames = await fileManager.listDir(PrevPathSegments.USER_AVATARS); const users = await knex('user_account').whereNotNull('avatar'); if (dirnames) { @@ -608,21 +615,23 @@ const upgradeUserAvatars = async () => { for (const dirname of dirnames) { const user = userByDirname[dirname]; - const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${dirname}`; + const dirPathSegment = `${PrevPathSegments.USER_AVATARS}/${dirname}`; if (user) { const size = await fileManager.getSize( `${dirPathSegment}/original.${user.avatar.extension}`, ); - await knex('user_account') - .update({ - avatar: knex.raw("?? || jsonb_build_object('sizeInBytes', ?::bigint)", [ - 'avatar', - size, - ]), - }) - .where('id', user.id); + if (size) { + await knex('user_account') + .update({ + avatar: knex.raw("?? || jsonb_build_object('sizeInBytes', ?::bigint)", [ + 'avatar', + size, + ]), + }) + .where('id', user.id); + } } else { await fileManager.deleteDir(dirPathSegment); } @@ -630,7 +639,7 @@ const upgradeUserAvatars = async () => { } for (const { avatar } of users) { - const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${avatar.dirname}`; + const dirPathSegment = `${PrevPathSegments.USER_AVATARS}/${avatar.dirname}`; const isExists = await fileManager.isExists(`${dirPathSegment}/cover-180.${avatar.extension}`); @@ -675,11 +684,11 @@ const upgradeBackgroundImages = async () => { const fileManager = sails.hooks['file-manager'].getInstance(); await fileManager.renameDir( - PROJECT_BACKGROUND_IMAGES_PATH_SEGMENT, - sails.config.custom.backgroundImagesPathSegment, + PrevPathSegments.PROJECT_BACKGROUND_IMAGES, + PrevPathSegments.BACKGROUND_IMAGES, ); - const dirnames = await fileManager.listDir(sails.config.custom.backgroundImagesPathSegment); + const dirnames = await fileManager.listDir(PrevPathSegments.BACKGROUND_IMAGES); const backgroundImages = await knex('background_image'); if (dirnames) { @@ -687,18 +696,20 @@ const upgradeBackgroundImages = async () => { for (const dirname of dirnames) { const backgroundImage = backgroundImageByDirname[dirname]; - const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${dirname}`; + const dirPathSegment = `${PrevPathSegments.BACKGROUND_IMAGES}/${dirname}`; if (backgroundImage) { const size = await fileManager.getSize( `${dirPathSegment}/original.${backgroundImage.extension}`, ); - await knex('background_image') - .update({ - size_in_bytes: size, - }) - .where('id', backgroundImage.id); + if (size) { + await knex('background_image') + .update({ + size_in_bytes: size, + }) + .where('id', backgroundImage.id); + } } else { await fileManager.deleteDir(dirPathSegment); } @@ -706,7 +717,7 @@ const upgradeBackgroundImages = async () => { } for (const backgroundImage of backgroundImages) { - const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${backgroundImage.dirname}`; + const dirPathSegment = `${PrevPathSegments.BACKGROUND_IMAGES}/${backgroundImage.dirname}`; const isExists = await fileManager.isExists( `${dirPathSegment}/outside-360.${backgroundImage.extension}`, @@ -755,7 +766,7 @@ const upgradeBackgroundImages = async () => { const upgradeFileAttachments = async () => { const fileManager = sails.hooks['file-manager'].getInstance(); - const dirnames = await fileManager.listDir(sails.config.custom.attachmentsPathSegment); + const dirnames = await fileManager.listDir(PrevPathSegments.ATTACHMENTS); const attachments = await knex('attachment').where('type', Attachment.Types.FILE); const fileReferenceIds = []; @@ -764,7 +775,7 @@ const upgradeFileAttachments = async () => { for (const dirname of dirnames) { const attachment = attachmentByDirname[dirname]; - const dirPathSegment = `${sails.config.custom.attachmentsPathSegment}/${dirname}`; + const dirPathSegment = `${PrevPathSegments.ATTACHMENTS}/${dirname}`; if (attachment) { if (uuid.validate(dirname)) { @@ -800,7 +811,7 @@ const upgradeFileAttachments = async () => { await fileManager.renameDir( `${dirPathSegment}`, - `${sails.config.custom.attachmentsPathSegment}/${id}`, + `${PrevPathSegments.ATTACHMENTS}/${id}`, ); return id; @@ -833,7 +844,7 @@ const upgradeFileAttachments = async () => { .whereRaw("??->>'image' IS NOT NULL", 'data'); for (const { data } of imageAttachments) { - const dirPathSegment = `${sails.config.custom.attachmentsPathSegment}/${data.fileReferenceId}`; + const dirPathSegment = `${PrevPathSegments.ATTACHMENTS}/${data.fileReferenceId}`; const thumbnailsPathSegment = `${dirPathSegment}/thumbnails`; const isExists = await fileManager.isExists( @@ -895,6 +906,22 @@ const upgradeFileAttachments = async () => { } }; +const upgradeDataStructure = async () => { + const fileManager = sails.hooks['file-manager'].getInstance(); + + await fileManager.renameDir(PrevPathSegments.FAVICONS, sails.config.custom.faviconsPathSegment); + + await fileManager.renameDir( + PrevPathSegments.USER_AVATARS, + sails.config.custom.userAvatarsPathSegment, + ); + + await fileManager.renameDir( + PrevPathSegments.BACKGROUND_IMAGES, + sails.config.custom.backgroundImagesPathSegment, + ); +}; + (async () => { try { let migrations; @@ -908,6 +935,7 @@ const upgradeFileAttachments = async () => { const isV1 = migrationNames[0] === '20180721020022_create_next_id_function.js'; const isLatestV1 = migrationNames.at(-1) === '20250131202710_add_list_color.js'; + const isInitialV2 = migrationNames.at(-1) === '20250228000022_version_2.js'; if (isV1 && !isLatestV1) { throw new Error('Update to latest v1 first'); @@ -919,9 +947,13 @@ const upgradeFileAttachments = async () => { await runStep('Upgrading database', upgradeDatabase); } - await runStep('Upgrading user avatars', upgradeUserAvatars); - await runStep('Upgrading background images', upgradeBackgroundImages); - await runStep('Upgrading file attachments', upgradeFileAttachments); + if (isV1 || isInitialV2) { + await runStep('Upgrading user avatars', upgradeUserAvatars); + await runStep('Upgrading background images', upgradeBackgroundImages); + await runStep('Upgrading file attachments', upgradeFileAttachments); + } + + await runStep('Upgrading data structure', upgradeDataStructure); } catch (error) { console.error(error); process.exitCode = 1; diff --git a/server/public/background-images/.gitkeep b/server/public/background-images/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/server/public/favicons/.gitkeep b/server/public/favicons/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/server/public/preloaded-favicons/.gitkeep b/server/public/preloaded-favicons/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/server/public/user-avatars/.gitkeep b/server/public/user-avatars/.gitkeep deleted file mode 100644 index e69de29b..00000000