feat: Track storage usage

server/api/helpers/attachments/delete-one.js

@@ -51,13 +51,11 @@ module.exports = {
       });
     }
 
-    const { attachment, fileReference } = await Attachment.qm.deleteOne(inputs.record.id, {
-      isFile: inputs.record.type === Attachment.Types.FILE,
-    });
+    const { attachment, uploadedFile } = await Attachment.qm.deleteOne(inputs.record.id);
 
     if (attachment) {
-      if (fileReference) {
-        sails.helpers.attachments.removeUnreferencedFiles(fileReference);
+      if (uploadedFile) {
+        sails.helpers.utils.removeUnreferencedUploadedFiles(uploadedFile);
       }
 
       sails.sockets.broadcast(

server/api/helpers/attachments/present-one.js

@@ -20,7 +20,7 @@ module.exports = {
         ...inputs.record,
         data: {
           ..._.omit(inputs.record.data, [
-            'fileReferenceId',
+            'uploadedFileId',
             'filename',
             'image.thumbnailsExtension',
           ]),

server/api/helpers/attachments/process-uploaded-file.js

@@ -10,7 +10,7 @@ const mime = require('mime');
 const sharp = require('sharp');
 
 const filenamify = require('../../../utils/filenamify');
-const { MAX_SIZE_IN_BYTES_TO_GET_ENCODING } = require('../../../constants');
+const { MAX_SIZE_TO_GET_ENCODING } = require('../../../constants');
 
 module.exports = {
   inputs: {
@@ -23,17 +23,22 @@ module.exports = {
   async fn(inputs) {
     const fileManager = sails.hooks['file-manager'].getInstance();
 
-    const { id: fileReferenceId } = await FileReference.create().fetch();
-    const dirPathSegment = `${sails.config.custom.attachmentsPathSegment}/${fileReferenceId}`;
     const filename = filenamify(inputs.file.filename);
-
     const mimeType = mime.getType(filename);
-    const sizeInBytes = inputs.file.size;
+    const { size } = inputs.file;
+
+    const { id: uploadedFileId } = await UploadedFile.qm.createOne({
+      mimeType,
+      size,
+      type: UploadedFile.Types.ATTACHMENT,
+    });
+
+    const dirPathSegment = `${sails.config.custom.attachmentsPathSegment}/${uploadedFileId}`;
 
     let buffer;
     let encoding = null;
 
-    if (sizeInBytes <= MAX_SIZE_IN_BYTES_TO_GET_ENCODING) {
+    if (size <= MAX_SIZE_TO_GET_ENCODING) {
       try {
         buffer = await fsPromises.readFile(inputs.file.fd);
       } catch (error) {
@@ -52,10 +57,10 @@ module.exports = {
     );
 
     const data = {
-      fileReferenceId,
+      uploadedFileId,
       filename,
       mimeType,
-      sizeInBytes,
+      size,
       encoding,
       image: null,
     };

server/api/helpers/attachments/remove-unreferenced-files.js

@@ -1,35 +0,0 @@
-/*!
- * Copyright (c) 2024 PLANKA Software GmbH
- * Licensed under the Fair Use License: https://github.com/plankanban/planka/blob/master/LICENSE.md
- */
-
-module.exports = {
-  sync: true,
-
-  inputs: {
-    fileReferenceOrFileReferences: {
-      type: 'ref',
-      required: true,
-    },
-  },
-
-  fn(inputs) {
-    const fileReferences = _.isPlainObject(inputs.fileReferenceOrFileReferences)
-      ? [inputs.fileReferenceOrFileReferences]
-      : inputs.fileReferenceOrFileReferences;
-
-    const fileManager = sails.hooks['file-manager'].getInstance();
-
-    fileReferences.forEach(async (fileReference) => {
-      if (fileReference.total !== null) {
-        return;
-      }
-
-      await fileManager.deleteDir(
-        `${sails.config.custom.attachmentsPathSegment}/${fileReference.id}`,
-      );
-
-      await FileReference.destroyOne(fileReference.id);
-    });
-  },
-};

server/api/helpers/background-images/delete-one.js

@@ -43,10 +43,10 @@ module.exports = {
       }
     }
 
-    const backgroundImage = await BackgroundImage.qm.deleteOne(inputs.record.id);
+    const { backgroundImage, uploadedFile } = await BackgroundImage.qm.deleteOne(inputs.record.id);
 
     if (backgroundImage) {
-      sails.helpers.backgroundImages.removeRelatedFiles(backgroundImage);
+      sails.helpers.utils.removeUnreferencedUploadedFiles(uploadedFile);
 
       const projectRelatedUserIds = await scoper.getProjectRelatedUserIds();
 

server/api/helpers/background-images/present-one.js

@@ -17,10 +17,10 @@ module.exports = {
     const fileManager = sails.hooks['file-manager'].getInstance();
 
     return {
-      ..._.omit(inputs.record, ['dirname', 'extension']),
-      url: `${fileManager.buildUrl(`${sails.config.custom.backgroundImagesPathSegment}/${inputs.record.dirname}/original.${inputs.record.extension}`)}`,
+      ..._.omit(inputs.record, ['uploadedFileId', 'extension']),
+      url: `${fileManager.buildUrl(`${sails.config.custom.backgroundImagesPathSegment}/${inputs.record.uploadedFileId}/original.${inputs.record.extension}`)}`,
       thumbnailUrls: {
-        outside360: `${fileManager.buildUrl(`${sails.config.custom.backgroundImagesPathSegment}/${inputs.record.dirname}/outside-360.${inputs.record.extension}`)}`,
+        outside360: `${fileManager.buildUrl(`${sails.config.custom.backgroundImagesPathSegment}/${inputs.record.uploadedFileId}/outside-360.${inputs.record.extension}`)}`,
       },
     };
   },

server/api/helpers/background-images/process-uploaded-file.js

@@ -3,9 +3,9 @@
  * Licensed under the Fair Use License: https://github.com/plankanban/planka/blob/master/LICENSE.md
  */
 
+const { v4: uuid } = require('uuid');
 const { rimraf } = require('rimraf');
 const mime = require('mime');
-const { v4: uuid } = require('uuid');
 const sharp = require('sharp');
 
 module.exports = {
@@ -32,8 +32,16 @@ module.exports = {
     });
 
     let metadata;
+    let originalBuffer;
+
     try {
       metadata = await image.metadata();
+
+      if (metadata.orientation && metadata.orientation > 4) {
+        image = image.rotate();
+      }
+
+      originalBuffer = await image.toBuffer();
     } catch (error) {
       await rimraf(inputs.file.fd);
       throw 'fileIsNotImage';
@@ -41,20 +49,19 @@ module.exports = {
 
     const fileManager = sails.hooks['file-manager'].getInstance();
 
-    const dirname = uuid();
-    const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${dirname}`;
-
-    if (metadata.orientation && metadata.orientation > 4) {
-      image = image.rotate();
-    }
-
     const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
+    const size = originalBuffer.length;
+
+    const { id: uploadedFileId } = await UploadedFile.qm.createOne({
+      mimeType,
+      size,
+      id: uuid(),
+      type: UploadedFile.Types.BACKGROUND_IMAGE,
+    });
+
+    const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${uploadedFileId}`;
 
-    let sizeInBytes;
     try {
-      const originalBuffer = await image.toBuffer();
-      sizeInBytes = originalBuffer.length;
-
       await fileManager.save(
         `${dirPathSegment}/original.${extension}`,
         originalBuffer,
@@ -82,6 +89,7 @@ module.exports = {
 
       await fileManager.deleteDir(dirPathSegment);
       await rimraf(inputs.file.fd);
+      await UploadedFile.qm.deleteOne(uploadedFileId);
 
       throw 'fileIsNotImage';
     }
@@ -89,9 +97,9 @@ module.exports = {
     await rimraf(inputs.file.fd);
 
     return {
-      dirname,
+      uploadedFileId,
       extension,
-      sizeInBytes,
+      size,
     };
   },
 };

server/api/helpers/background-images/remove-related-files.js

@@ -1,29 +0,0 @@
-/*!
- * Copyright (c) 2024 PLANKA Software GmbH
- * Licensed under the Fair Use License: https://github.com/plankanban/planka/blob/master/LICENSE.md
- */
-
-module.exports = {
-  sync: true,
-
-  inputs: {
-    recordOrRecords: {
-      type: 'ref',
-      required: true,
-    },
-  },
-
-  fn(inputs) {
-    const backgroundImages = _.isPlainObject(inputs.recordOrRecords)
-      ? [inputs.recordOrRecords]
-      : inputs.recordOrRecords;
-
-    const fileManager = sails.hooks['file-manager'].getInstance();
-
-    backgroundImages.forEach(async (backgroundImage) => {
-      await fileManager.deleteDir(
-        `${sails.config.custom.backgroundImagesPathSegment}/${backgroundImage.dirname}`,
-      );
-    });
-  },
-};

server/api/helpers/cards/delete-related.js

@@ -48,11 +48,11 @@ module.exports = {
       },
     );
 
-    const { fileReferences } = await Attachment.qm.delete({
+    const { uploadedFiles } = await Attachment.qm.delete({
       cardId: cardIdOrIds,
     });
 
-    sails.helpers.attachments.removeUnreferencedFiles(fileReferences);
+    sails.helpers.utils.removeUnreferencedUploadedFiles(uploadedFiles);
 
     const customFieldGroups = await CustomFieldGroup.qm.delete({
       cardId: cardIdOrIds,

server/api/helpers/projects/delete-related.js

@@ -29,11 +29,11 @@ module.exports = {
       projectId: projectIdOrIds,
     });
 
-    const backgroundImages = await BackgroundImage.qm.delete({
+    const { uploadedFiles } = await BackgroundImage.qm.delete({
       projectId: projectIdOrIds,
     });
 
-    sails.helpers.backgroundImages.removeRelatedFiles(backgroundImages);
+    sails.helpers.utils.removeUnreferencedUploadedFiles(uploadedFiles);
 
     const baseCustomFieldGroups = await BaseCustomFieldGroup.qm.delete({
       projectId: projectIdOrIds,

server/api/helpers/users/delete-one.js

@@ -23,10 +23,12 @@ module.exports = {
       inputs.record,
     );
 
-    const user = await User.qm.deleteOne(inputs.record.id);
+    const { user, uploadedFile } = await User.qm.deleteOne(inputs.record.id);
 
     if (user) {
-      sails.helpers.users.removeRelatedFiles(user);
+      if (uploadedFile) {
+        sails.helpers.utils.removeUnreferencedUploadedFiles(uploadedFile);
+      }
 
       const scoper = sails.helpers.users.makeScoper(user);
       scoper.boardMemberships = boardMemberships;

server/api/helpers/users/present-one.js

@@ -28,9 +28,9 @@ module.exports = {
         'termsAcceptedAt',
       ]),
       avatar: inputs.record.avatar && {
-        url: `${fileManager.buildUrl(`${sails.config.custom.userAvatarsPathSegment}/${inputs.record.avatar.dirname}/original.${inputs.record.avatar.extension}`)}`,
+        url: `${fileManager.buildUrl(`${sails.config.custom.userAvatarsPathSegment}/${inputs.record.avatar.uploadedFileId}/original.${inputs.record.avatar.extension}`)}`,
         thumbnailUrls: {
-          cover180: `${fileManager.buildUrl(`${sails.config.custom.userAvatarsPathSegment}/${inputs.record.avatar.dirname}/cover-180.${inputs.record.avatar.extension}`)}`,
+          cover180: `${fileManager.buildUrl(`${sails.config.custom.userAvatarsPathSegment}/${inputs.record.avatar.uploadedFileId}/cover-180.${inputs.record.avatar.extension}`)}`,
         },
       },
       termsType: sails.hooks.terms.getTypeByUserRole(inputs.record.role),

server/api/helpers/users/process-uploaded-avatar-file.js

@@ -3,9 +3,9 @@
  * Licensed under the Fair Use License: https://github.com/plankanban/planka/blob/master/LICENSE.md
  */
 
+const { v4: uuid } = require('uuid');
 const { rimraf } = require('rimraf');
 const mime = require('mime');
-const { v4: uuid } = require('uuid');
 const sharp = require('sharp');
 
 module.exports = {
@@ -32,8 +32,16 @@ module.exports = {
     });
 
     let metadata;
+    let originalBuffer;
+
     try {
       metadata = await image.metadata();
+
+      if (metadata.orientation && metadata.orientation > 4) {
+        image = image.rotate();
+      }
+
+      originalBuffer = await image.toBuffer();
     } catch (error) {
       await rimraf(inputs.file.fd);
       throw 'fileIsNotImage';
@@ -41,20 +49,19 @@ module.exports = {
 
     const fileManager = sails.hooks['file-manager'].getInstance();
 
-    const dirname = uuid();
-    const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${dirname}`;
-
-    if (metadata.orientation && metadata.orientation > 4) {
-      image = image.rotate();
-    }
-
     const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format;
+    const size = originalBuffer.length;
+
+    const { id: uploadedFileId } = await UploadedFile.qm.createOne({
+      mimeType,
+      size,
+      id: uuid(),
+      type: UploadedFile.Types.USER_AVATAR,
+    });
+
+    const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${uploadedFileId}`;
 
-    let sizeInBytes;
     try {
-      const originalBuffer = await image.toBuffer();
-      sizeInBytes = originalBuffer.length;
-
       await fileManager.save(
         `${dirPathSegment}/original.${extension}`,
         originalBuffer,
@@ -81,6 +88,7 @@ module.exports = {
 
       await fileManager.deleteDir(dirPathSegment);
       await rimraf(inputs.file.fd);
+      await UploadedFile.qm.deleteOne(uploadedFileId);
 
       throw 'fileIsNotImage';
     }
@@ -88,9 +96,9 @@ module.exports = {
     await rimraf(inputs.file.fd);
 
     return {
-      dirname,
+      uploadedFileId,
       extension,
-      sizeInBytes,
+      size,
     };
   },
 };

server/api/helpers/users/update-one.js

@@ -69,8 +69,10 @@ module.exports = {
     }
 
     let user;
+    let uploadedFile;
+
     try {
-      user = await User.qm.updateOne(inputs.record.id, values);
+      ({ user, uploadedFile } = await User.qm.updateOne(inputs.record.id, values));
     } catch (error) {
       if (error.code === 'E_UNIQUE') {
         throw 'emailAlreadyInUse';
@@ -91,10 +93,8 @@ module.exports = {
     }
 
     if (user) {
-      if (inputs.record.avatar) {
-        if (!user.avatar || user.avatar.dirname !== inputs.record.avatar.dirname) {
-          sails.helpers.users.removeRelatedFiles(inputs.record);
-        }
+      if (uploadedFile) {
+        sails.helpers.utils.removeUnreferencedUploadedFiles(uploadedFile);
       }
 
       if (!_.isUndefined(values.password) || isDeactivatedChangeToTrue) {

server/api/helpers/utils/download-favicon.js

@@ -8,7 +8,7 @@ const icoToPng = require('ico-to-png');
 const sharp = require('sharp');
 
 const FETCH_TIMEOUT = 4000;
-const MAX_RESPONSE_LENGTH_IN_BYTES = 1024 * 1024;
+const MAX_RESPONSE_LENGTH = 1024 * 1024;
 
 const FAVICON_TAGS_REGEX = /<link [^>]*rel="([^"]* )?icon( [^"]*)?"[^>]*>/gi;
 const HREF_REGEX = /href="(.*?)"/i;
@@ -39,7 +39,7 @@ const readResponse = async (response) => {
     chunks.push(value);
     receivedLength += value.length;
 
-    if (receivedLength > MAX_RESPONSE_LENGTH_IN_BYTES) {
+    if (receivedLength > MAX_RESPONSE_LENGTH) {
       reader.cancel();
 
       return {
@@ -133,6 +133,12 @@ module.exports = {
       return;
     }
 
+    const availableStorage = await sails.helpers.utils.getAvailableStorage();
+
+    if (availableStorage !== null && readedResponse.buffer.length >= availableStorage) {
+      return;
+    }
+
     let image = sharp(readedResponse.buffer);
 
     let metadata;

server/api/helpers/utils/get-available-storage.js

@@ -0,0 +1,17 @@
+/*!
+ * Copyright (c) 2024 PLANKA Software GmbH
+ * Licensed under the Fair Use License: https://github.com/plankanban/planka/blob/master/LICENSE.md
+ */
+
+module.exports = {
+  async fn() {
+    const { storageLimit } = sails.config.custom;
+
+    if (_.isNil(storageLimit)) {
+      return null;
+    }
+
+    const storageUsage = await StorageUsage.qm.getOneMain();
+    return BigInt(storageLimit) - BigInt(storageUsage.total);
+  },
+};

server/api/helpers/utils/receive-file.js

@@ -4,37 +4,55 @@
  */
 
 const util = require('util');
-const { v4: uuid } = require('uuid');
 
 module.exports = {
-  friendlyName: 'Receive uploaded file from request',
-
-  description:
-    'Store a file uploaded from a MIME-multipart request part. The resulting file will have a unique UUID-based name with the same extension.',
-
   inputs: {
-    paramName: {
-      type: 'string',
-      required: true,
-      description: 'The MIME multi-part parameter containing the file to receive.',
-    },
-    req: {
+    file: {
       type: 'ref',
       required: true,
-      description: 'The request to receive the file from.',
+    },
+    enforceStorageLimit: {
+      type: 'boolean',
+      defaultsTo: true,
     },
   },
 
   async fn(inputs, exits) {
+    const { maxUploadFileSize } = sails.config.custom;
+
+    let availableStorage = null;
+    if (inputs.enforceStorageLimit) {
+      availableStorage = await sails.helpers.utils.getAvailableStorage();
+    }
+
+    let maxBytes = _.isNil(maxUploadFileSize) ? null : maxUploadFileSize;
+    if (availableStorage !== null) {
+      if (maxBytes) {
+        maxBytes = availableStorage < maxBytes ? availableStorage : maxBytes;
+      } else {
+        maxBytes = availableStorage;
+      }
+    }
+
     const upload = util.promisify((options, callback) =>
-      inputs.req.file(inputs.paramName).upload(options, (error, files) => callback(error, files)),
+      inputs.file.upload(options, (error, files) => {
+        if (
+          error &&
+          error.code === 'E_EXCEEDS_UPLOAD_LIMIT' &&
+          availableStorage !== null &&
+          (_.isNil(maxUploadFileSize) || error.maxBytes < maxUploadFileSize)
+        ) {
+          return callback(new Error('Storage limit reached'), files);
+        }
+
+        return callback(error, files);
+      }),
     );
 
     return exits.success(
       await upload({
+        maxBytes,
         dirname: sails.config.custom.uploadsTempPath,
-        saveAs: uuid(),
-        maxBytes: null,
       }),
     );
   },

server/api/helpers/utils/remove-unreferenced-uploaded-files.js

@@ -0,0 +1,41 @@
+/*!
+ * Copyright (c) 2024 PLANKA Software GmbH
+ * Licensed under the Fair Use License: https://github.com/plankanban/planka/blob/master/LICENSE.md
+ */
+
+const { Types } = require('../../models/UploadedFile');
+
+const PATH_SEGMENT_BY_TYPE = {
+  [Types.USER_AVATAR]: sails.config.custom.userAvatarsPathSegment,
+  [Types.BACKGROUND_IMAGE]: sails.config.custom.backgroundImagesPathSegment,
+  [Types.ATTACHMENT]: sails.config.custom.attachmentsPathSegment,
+};
+
+module.exports = {
+  sync: true,
+
+  inputs: {
+    uploadedFileOrUploadedFiles: {
+      type: 'ref',
+      required: true,
+    },
+  },
+
+  fn(inputs) {
+    const uploadedFiles = _.isPlainObject(inputs.uploadedFileOrUploadedFiles)
+      ? [inputs.uploadedFileOrUploadedFiles]
+      : inputs.uploadedFileOrUploadedFiles;
+
+    const fileManager = sails.hooks['file-manager'].getInstance();
+
+    // TODO: optimize?
+    uploadedFiles.forEach(async (uploadedFile) => {
+      if (uploadedFile.referencesTotal !== null) {
+        return;
+      }
+
+      await fileManager.deleteDir(`${PATH_SEGMENT_BY_TYPE[uploadedFile.type]}/${uploadedFile.id}`);
+      await UploadedFile.qm.deleteOne(uploadedFile.id);
+    });
+  },
+};