feat: Migrate file storage to unified data directory

.dockerignore

@@ -11,24 +11,9 @@ server/test
 server/.tmp
 server/.venv
 
-server/views/*
+server/views/index.ejs
-!server/views/.gitkeep
 
-server/public/*
+server/data/*
-!server/public/preloaded-favicons
+!server/data/.gitkeep
-!server/public/favicons
-server/public/favicons/*
-!server/public/favicons/.gitkeep
-!server/public/user-avatars
-server/public/user-avatars/*
-!server/public/user-avatars/.gitkeep
-!server/public/background-images
-server/public/background-images/*
-!server/public/background-images/.gitkeep
-
-server/private/*
-!server/private/attachments
-server/private/attachments/*
-!server/private/attachments/.gitkeep
 
 client/dist

Dockerfile

@@ -49,11 +49,7 @@ RUN python3 -m venv .venv \
   && mv .env.sample .env \
   && npm config set update-notifier false
 
-VOLUME /app/public/favicons
+VOLUME /app/data
-VOLUME /app/public/user-avatars
-VOLUME /app/public/background-images
-VOLUME /app/private/attachments
-
 EXPOSE 1337
 
 HEALTHCHECK --interval=10s --timeout=2s --start-period=15s \

charts/planka/templates/deployment.yaml

@@ -54,17 +54,8 @@ spec:
               path: /
               port: http
           volumeMounts:
-            - mountPath: /app/public/favicons
+            - mountPath: /app/data
-              subPath: favicons
+              subPath: data
-              name: planka
-            - mountPath: /app/public/user-avatars
-              subPath: user-avatars
-              name: planka
-            - mountPath: /app/public/background-images
-              subPath: background-images
-              name: planka
-            - mountPath: /app/private/attachments
-              subPath: attachments
               name: planka
           {{- if .Values.securityContext.readOnlyRootFilesystem }}
             - mountPath: /app/logs

docker-backup.sh

@@ -22,28 +22,16 @@ echo -n "Exporting postgres database ... "
 docker exec -t "$PLANKA_DOCKER_CONTAINER_POSTGRES" pg_dumpall -c -U postgres > "$BACKUP_DATETIME-backup/postgres.sql"
 echo "Success!"
 
-# Export Docker Voumes
+# Export Docker Volume
-echo -n "Exporting favicons ... "
+echo -n "Exporting data volume ... "
-docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/public/favicons /backup/favicons
+docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/data /backup/data
-echo "Success!"
-echo -n "Exporting user-avatars ... "
-docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/public/user-avatars /backup/user-avatars
-echo "Success!"
-echo -n "Exporting background-images ... "
-docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/public/background-images /backup/background-images
-echo "Success!"
-echo -n "Exporting attachments ... "
-docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$BACKUP_DATETIME-backup:/backup" ubuntu cp -r /app/private/attachments /backup/attachments
 echo "Success!"
 
 # Create tgz
 echo -n "Creating final tarball $BACKUP_DATETIME-backup.tgz ... "
 tar -czf "$BACKUP_DATETIME-backup.tgz" \
     "$BACKUP_DATETIME-backup/postgres.sql" \
-    "$BACKUP_DATETIME-backup/favicons" \
+    "$BACKUP_DATETIME-backup/data"
-    "$BACKUP_DATETIME-backup/user-avatars" \
-    "$BACKUP_DATETIME-backup/background-images" \
-    "$BACKUP_DATETIME-backup/attachments"
 echo "Success!"
 
 # Remove source files

docker-compose.yml

@@ -3,10 +3,7 @@ services:
     image: ghcr.io/plankanban/planka:2.0.0-rc.4
     restart: on-failure
     volumes:
-      - favicons:/app/public/favicons
+      - data:/app/data
-      - user-avatars:/app/public/user-avatars
-      - background-images:/app/public/background-images
-      - attachments:/app/private/attachments
     # Optionally override this to your user/group
     # user: 1000:1000
     # tmpfs:
@@ -132,8 +129,5 @@ services:
       retries: 5
 
 volumes:
-  favicons:
+  data:
-  user-avatars:
-  background-images:
-  attachments:
   db-data:

docker-restore.sh

@@ -19,18 +19,9 @@ echo -n "Importing postgres database ... "
 cat "$PLANKA_BACKUP_ARCHIVE/postgres.sql" | docker exec -i "$PLANKA_DOCKER_CONTAINER_POSTGRES" psql -U postgres
 echo "Success!"
 
-# Restore Docker Volumes
+# Restore Docker Volume
-echo -n "Importing favicons ... "
+echo -n "Importing data volume ... "
-docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/favicons /app/public/
+docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/data/. /app/public/data
-echo "Success!"
-echo -n "Importing user-avatars ... "
-docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/user-avatars /app/public/
-echo "Success!"
-echo -n "Importing background-images ... "
-docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/background-images /app/public/
-echo "Success!"
-echo -n "Importing attachments ... "
-docker run --rm --volumes-from "$PLANKA_DOCKER_CONTAINER_PLANKA" -v "$(pwd)/$PLANKA_BACKUP_ARCHIVE:/backup" ubuntu cp -rf /backup/attachments /app/private/
 echo "Success!"
 
 echo -n "Cleaning up temporary files and folders ... "

server/.buildignore

@@ -20,17 +20,6 @@ test
 .tmp
 .venv
 
-views/*
+views/index.ejs
 
-public/*
+data/*
-!public/preloaded-favicons
-!public/favicons
-public/favicons/*
-!public/user-avatars
-public/user-avatars/*
-!public/background-images
-public/background-images/*
-
-private/*
-!private/attachments
-private/attachments/*

server/.gitignore

@@ -134,22 +134,7 @@ swagger.json
 dist
 logs
 
-views/*
+views/index.ejs
-!views/.gitkeep
 
-public/*
+data/*
-!public/preloaded-favicons
+!data/.gitkeep
-!public/favicons
-public/favicons/*
-!public/favicons/.gitkeep
-!public/user-avatars
-public/user-avatars/*
-!public/user-avatars/.gitkeep
-!public/background-images
-public/background-images/*
-!public/background-images/.gitkeep
-
-private/*
-!private/attachments
-private/attachments/*
-!private/attachments/.gitkeep

server/api/helpers/utils/is-preloaded-favicon-exists.js

@@ -7,12 +7,7 @@ const fs = require('fs');
 const path = require('path');
 
 const PRELOADED_FAVICON_FILENAMES = fs
-  .readdirSync(
+  .readdirSync(path.join(sails.config.paths.public, 'preloaded-favicons'))
-    path.join(
-      sails.config.custom.uploadsBasePath,
-      sails.config.custom.preloadedFaviconsPathSegment,
-    ),
-  )
   .filter((filename) => filename.endsWith('.png'));
 
 const PRELOADED_FAVICON_HOSTNAMES_SET = new Set(

server/config/custom.js

@@ -8,6 +8,7 @@
  * https://sailsjs.com/config/custom
  */
 
+const path = require('path');
 const { URL } = require('url');
 const bytes = require('bytes');
 const sails = require('sails');
@@ -48,12 +49,11 @@ module.exports.custom = {
 
   // Location to receive uploaded files in. Default (non-string value) is a Sails-specific location.
   uploadsTempPath: null,
-  uploadsBasePath: sails.config.appPath,
+  uploadsBasePath: path.join(sails.config.appPath, 'data'),
 
-  preloadedFaviconsPathSegment: 'public/preloaded-favicons',
+  faviconsPathSegment: 'protected/favicons',
-  faviconsPathSegment: 'public/favicons',
+  userAvatarsPathSegment: 'protected/user-avatars',
-  userAvatarsPathSegment: 'public/user-avatars',
+  backgroundImagesPathSegment: 'protected/background-images',
-  backgroundImagesPathSegment: 'public/background-images',
   attachmentsPathSegment: 'private/attachments',
 
   defaultAdminEmail:

server/config/routes.js

@@ -80,13 +80,13 @@ const serveStatic = async (prefix, getPathSegment, req, res) => {
   return readStream.pipe(res);
 };
 
-const publicStaticDirServer = (prefix, getPathSegment) => (req, res, next) => {
+/* const publicStaticDirServer = (prefix, getPathSegment) => (req, res, next) => {
   if (!req.url.startsWith(prefix)) {
     return next();
   }
 
   return serveStatic(prefix, getPathSegment, req, res);
-};
+}; */
 
 const protectedStaticDirServer = (prefix, getPathSegment) => (req, res, next) => {
   if (!req.url.startsWith(prefix)) {
@@ -235,14 +235,6 @@ module.exports.routes = {
 
   'PATCH /api/_internal/config': '_internal/update-config',
 
-  'GET /preloaded-favicons/*': {
-    fn: publicStaticDirServer(
-      '/preloaded-favicons',
-      () => sails.config.custom.preloadedFaviconsPathSegment,
-    ),
-    skipAssets: false,
-  },
-
   'GET /favicons/*': {
     fn: protectedStaticDirServer('/favicons', () => sails.config.custom.faviconsPathSegment),
     skipAssets: false,

server/db/upgrade.js

@@ -24,7 +24,14 @@ const PrevActionTypes = {
   COMMENT_CARD: 'commentCard',
 };
 
-const PROJECT_BACKGROUND_IMAGES_PATH_SEGMENT = 'public/project-background-images';
+const PrevPathSegments = {
+  PROJECT_BACKGROUND_IMAGES: 'public/project-background-images',
+
+  FAVICONS: 'public/favicons',
+  USER_AVATARS: 'public/user-avatars',
+  BACKGROUND_IMAGES: 'public/background-images',
+  ATTACHMENTS: 'private/attachments',
+};
 
 const readStreamToBuffer = (readStream) =>
   new Promise((resolve, reject) => {
@@ -600,7 +607,7 @@ const upgradeDatabase = async () => {
 const upgradeUserAvatars = async () => {
   const fileManager = sails.hooks['file-manager'].getInstance();
 
-  const dirnames = await fileManager.listDir(sails.config.custom.userAvatarsPathSegment);
+  const dirnames = await fileManager.listDir(PrevPathSegments.USER_AVATARS);
   const users = await knex('user_account').whereNotNull('avatar');
 
   if (dirnames) {
@@ -608,21 +615,23 @@ const upgradeUserAvatars = async () => {
 
     for (const dirname of dirnames) {
       const user = userByDirname[dirname];
-      const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${dirname}`;
+      const dirPathSegment = `${PrevPathSegments.USER_AVATARS}/${dirname}`;
 
       if (user) {
         const size = await fileManager.getSize(
           `${dirPathSegment}/original.${user.avatar.extension}`,
         );
 
-        await knex('user_account')
+        if (size) {
-          .update({
+          await knex('user_account')
-            avatar: knex.raw("?? || jsonb_build_object('sizeInBytes', ?::bigint)", [
+            .update({
-              'avatar',
+              avatar: knex.raw("?? || jsonb_build_object('sizeInBytes', ?::bigint)", [
-              size,
+                'avatar',
-            ]),
+                size,
-          })
+              ]),
-          .where('id', user.id);
+            })
+            .where('id', user.id);
+        }
       } else {
         await fileManager.deleteDir(dirPathSegment);
       }
@@ -630,7 +639,7 @@ const upgradeUserAvatars = async () => {
   }
 
   for (const { avatar } of users) {
-    const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${avatar.dirname}`;
+    const dirPathSegment = `${PrevPathSegments.USER_AVATARS}/${avatar.dirname}`;
 
     const isExists = await fileManager.isExists(`${dirPathSegment}/cover-180.${avatar.extension}`);
 
@@ -675,11 +684,11 @@ const upgradeBackgroundImages = async () => {
   const fileManager = sails.hooks['file-manager'].getInstance();
 
   await fileManager.renameDir(
-    PROJECT_BACKGROUND_IMAGES_PATH_SEGMENT,
+    PrevPathSegments.PROJECT_BACKGROUND_IMAGES,
-    sails.config.custom.backgroundImagesPathSegment,
+    PrevPathSegments.BACKGROUND_IMAGES,
   );
 
-  const dirnames = await fileManager.listDir(sails.config.custom.backgroundImagesPathSegment);
+  const dirnames = await fileManager.listDir(PrevPathSegments.BACKGROUND_IMAGES);
   const backgroundImages = await knex('background_image');
 
   if (dirnames) {
@@ -687,18 +696,20 @@ const upgradeBackgroundImages = async () => {
 
     for (const dirname of dirnames) {
       const backgroundImage = backgroundImageByDirname[dirname];
-      const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${dirname}`;
+      const dirPathSegment = `${PrevPathSegments.BACKGROUND_IMAGES}/${dirname}`;
 
       if (backgroundImage) {
         const size = await fileManager.getSize(
           `${dirPathSegment}/original.${backgroundImage.extension}`,
         );
 
-        await knex('background_image')
+        if (size) {
-          .update({
+          await knex('background_image')
-            size_in_bytes: size,
+            .update({
-          })
+              size_in_bytes: size,
-          .where('id', backgroundImage.id);
+            })
+            .where('id', backgroundImage.id);
+        }
       } else {
         await fileManager.deleteDir(dirPathSegment);
       }
@@ -706,7 +717,7 @@ const upgradeBackgroundImages = async () => {
   }
 
   for (const backgroundImage of backgroundImages) {
-    const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${backgroundImage.dirname}`;
+    const dirPathSegment = `${PrevPathSegments.BACKGROUND_IMAGES}/${backgroundImage.dirname}`;
 
     const isExists = await fileManager.isExists(
       `${dirPathSegment}/outside-360.${backgroundImage.extension}`,
@@ -755,7 +766,7 @@ const upgradeBackgroundImages = async () => {
 const upgradeFileAttachments = async () => {
   const fileManager = sails.hooks['file-manager'].getInstance();
 
-  const dirnames = await fileManager.listDir(sails.config.custom.attachmentsPathSegment);
+  const dirnames = await fileManager.listDir(PrevPathSegments.ATTACHMENTS);
   const attachments = await knex('attachment').where('type', Attachment.Types.FILE);
 
   const fileReferenceIds = [];
@@ -764,7 +775,7 @@ const upgradeFileAttachments = async () => {
 
     for (const dirname of dirnames) {
       const attachment = attachmentByDirname[dirname];
-      const dirPathSegment = `${sails.config.custom.attachmentsPathSegment}/${dirname}`;
+      const dirPathSegment = `${PrevPathSegments.ATTACHMENTS}/${dirname}`;
 
       if (attachment) {
         if (uuid.validate(dirname)) {
@@ -800,7 +811,7 @@ const upgradeFileAttachments = async () => {
 
             await fileManager.renameDir(
               `${dirPathSegment}`,
-              `${sails.config.custom.attachmentsPathSegment}/${id}`,
+              `${PrevPathSegments.ATTACHMENTS}/${id}`,
             );
 
             return id;
@@ -833,7 +844,7 @@ const upgradeFileAttachments = async () => {
     .whereRaw("??->>'image' IS NOT NULL", 'data');
 
   for (const { data } of imageAttachments) {
-    const dirPathSegment = `${sails.config.custom.attachmentsPathSegment}/${data.fileReferenceId}`;
+    const dirPathSegment = `${PrevPathSegments.ATTACHMENTS}/${data.fileReferenceId}`;
     const thumbnailsPathSegment = `${dirPathSegment}/thumbnails`;
 
     const isExists = await fileManager.isExists(
@@ -895,6 +906,22 @@ const upgradeFileAttachments = async () => {
   }
 };
 
+const upgradeDataStructure = async () => {
+  const fileManager = sails.hooks['file-manager'].getInstance();
+
+  await fileManager.renameDir(PrevPathSegments.FAVICONS, sails.config.custom.faviconsPathSegment);
+
+  await fileManager.renameDir(
+    PrevPathSegments.USER_AVATARS,
+    sails.config.custom.userAvatarsPathSegment,
+  );
+
+  await fileManager.renameDir(
+    PrevPathSegments.BACKGROUND_IMAGES,
+    sails.config.custom.backgroundImagesPathSegment,
+  );
+};
+
 (async () => {
   try {
     let migrations;
@@ -908,6 +935,7 @@ const upgradeFileAttachments = async () => {
 
     const isV1 = migrationNames[0] === '20180721020022_create_next_id_function.js';
     const isLatestV1 = migrationNames.at(-1) === '20250131202710_add_list_color.js';
+    const isInitialV2 = migrationNames.at(-1) === '20250228000022_version_2.js';
 
     if (isV1 && !isLatestV1) {
       throw new Error('Update to latest v1 first');
@@ -919,9 +947,13 @@ const upgradeFileAttachments = async () => {
       await runStep('Upgrading database', upgradeDatabase);
     }
 
-    await runStep('Upgrading user avatars', upgradeUserAvatars);
+    if (isV1 || isInitialV2) {
-    await runStep('Upgrading background images', upgradeBackgroundImages);
+      await runStep('Upgrading user avatars', upgradeUserAvatars);
-    await runStep('Upgrading file attachments', upgradeFileAttachments);
+      await runStep('Upgrading background images', upgradeBackgroundImages);
+      await runStep('Upgrading file attachments', upgradeFileAttachments);
+    }
+
+    await runStep('Upgrading data structure', upgradeDataStructure);
   } catch (error) {
     console.error(error);
     process.exitCode = 1;