starred/planka
1
0
Fork 0
mirror of https://github.com/plankanban/planka.git synced 2025-12-25 17:25:01 +03:00
Code Issues 418 Packages Projects Releases 12 Wiki Activity

feat: Track storage usage

Browse Source
This commit is contained in:
Maksim Eltyshev
2025-08-23 00:03:20 +02:00
parent 2f4bcb0583
commit 4d77a1f596
89 changed files with 1052 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
server/db/migrations/20250820144730_track_storage_usage.js Normal file
View File

@@ -0,0 +1,161 @@
/*!
* Copyright (c) 2024 PLANKA Software GmbH
* Licensed under the Fair Use License: https://github.com/plankanban/planka/blob/master/LICENSE.md
*/
const mime = require('mime');
exports.up = async (knex) => {
await knex.schema.createTable('storage_usage', (table) => {
/* Columns */
table.bigInteger('id').primary().defaultTo(knex.raw('next_id()'));
table.bigInteger('total').notNullable();
table.bigInteger('user_avatars').notNullable();
table.bigInteger('background_images').notNullable();
table.bigInteger('attachments').notNullable();
table.timestamp('created_at', true);
table.timestamp('updated_at', true);
});
await knex.schema.alterTable('file_reference', (table) => {
table.dropPrimary();
table.dropIndex('total');
});
await knex.schema.renameTable('file_reference', 'uploaded_file');
await knex.schema.alterTable('uploaded_file', (table) => {
/* Columns */
table.text('type').notNullable().defaultTo('attachment');
table.text('mime_type');
table.bigInteger('size').notNullable().defaultTo(0);
/* Modifications */
table.text('id').primary().defaultTo(knex.raw('next_id()')).alter();
table.renameColumn('total', 'references_total');
/* Indexes */
table.index('type');
table.index('references_total');
});
await knex.schema.alterTable('uploaded_file', (table) => {
table.text('type').notNullable().alter();
table.bigInteger('size').notNullable().alter();
});
await knex.raw(`
UPDATE user_account
SET avatar = avatar - 'dirname' - 'sizeInBytes' || jsonb_build_object('uploadedFileId', avatar->'dirname', 'size', avatar->'sizeInBytes')
WHERE avatar IS NOT NULL;
`);
await knex.schema.alterTable('background_image', (table) => {
table.renameColumn('dirname', 'uploaded_file_id');
table.renameColumn('size_in_bytes', 'size');
});
await knex.raw(`
UPDATE attachment
SET data = data - 'fileReferenceId' - 'sizeInBytes' || jsonb_build_object('uploadedFileId', data->'fileReferenceId', 'size', data->'sizeInBytes')
WHERE type = 'file';
`);
await knex.raw(`
UPDATE uploaded_file
SET
type = 'attachment',
mime_type = attachment.data->>'mimeType',
size = (attachment.data->>'size')::bigint
FROM attachment
WHERE (attachment.data->>'uploadedFileId')::text = uploaded_file.id AND attachment.type = 'file';
`);
const users = await knex('user_account').whereNotNull('avatar');
const createdAt = new Date().toISOString();
await knex.batchInsert(
'uploaded_file',
users.map(({ avatar }) => ({
createdAt,
id: avatar.uploadedFileId,
type: 'userAvatar',
referencesTotal: 1,
mimeType: mime.getType(avatar.extension),
size: avatar.size,
})),
);
const backgroundImages = await knex('background_image');
await knex.batchInsert(
'uploaded_file',
backgroundImages.map((backgroundImage) => ({
id: backgroundImage.uploaded_file_id,
type: 'backgroundImage',
referencesTotal: 1,
mimeType: mime.getType(backgroundImage.extension),
size: backgroundImage.size,
createdAt: backgroundImage.created_at,
})),
);
return knex.raw(`
INSERT INTO storage_usage (id, total, user_avatars, background_images, attachments, created_at)
SELECT
1 AS id,
COALESCE(SUM(size), 0) AS total,
COALESCE(SUM(CASE WHEN type = 'userAvatar' THEN size ELSE 0 END), 0) AS user_avatars,
COALESCE(SUM(CASE WHEN type = 'backgroundImage' THEN size ELSE 0 END), 0) AS background_images,
COALESCE(SUM(CASE WHEN type = 'attachment' THEN size ELSE 0 END), 0) AS attachments,
timezone('UTC', now()) AS created_at
FROM uploaded_file;
`);
};
exports.down = async (knex) => {
await knex.schema.dropTable('storage_usage');
await knex('uploaded_file').delete().whereNot('type', 'attachment');
await knex.schema.alterTable('uploaded_file', (table) => {
table.dropPrimary();
table.dropIndex('references_total');
});
await knex.schema.renameTable('uploaded_file', 'file_reference');
await knex.schema.alterTable('file_reference', (table) => {
table.dropColumn('type');
table.dropColumn('mime_type');
table.dropColumn('size');
table.bigInteger('id').primary().defaultTo(knex.raw('next_id()')).alter();
table.renameColumn('references_total', 'total');
table.index('total');
});
await knex.raw(`
UPDATE user_account
SET avatar = avatar - 'uploadedFileId' - 'size' || jsonb_build_object('dirname', avatar->'uploadedFileId', 'sizeInBytes', avatar->'size')
WHERE avatar IS NOT NULL;
`);
await knex.schema.alterTable('background_image', (table) => {
table.renameColumn('uploaded_file_id', 'dirname');
table.renameColumn('size', 'size_in_bytes');
});
return knex.raw(`
UPDATE attachment
SET data = data - 'uploadedFileId' - 'size' || jsonb_build_object('fileReferenceId', data->'uploadedFileId', 'sizeInBytes', data->'size')
WHERE type = 'file';
`);
};

18
server/db/upgrade.js
View File

@@ -18,7 +18,7 @@ const rc = require('sails/accessible/rc');
const _ = require('lodash');
const knexfile = require('./knexfile');
const { MAX_SIZE_IN_BYTES_TO_GET_ENCODING, POSITION_GAP } = require('../constants');
const { MAX_SIZE_TO_GET_ENCODING, POSITION_GAP } = require('../constants');
const PrevActionTypes = {
COMMENT_CARD: 'commentCard',
@@ -611,7 +611,7 @@ const upgradeUserAvatars = async () => {
const dirPathSegment = `${sails.config.custom.userAvatarsPathSegment}/${dirname}`;
if (user) {
const sizeInBytes = await fileManager.getSizeInBytes(
const size = await fileManager.getSize(
`${dirPathSegment}/original.${user.avatar.extension}`,
);
@@ -619,7 +619,7 @@ const upgradeUserAvatars = async () => {
.update({
avatar: knex.raw("?? || jsonb_build_object('sizeInBytes', ?::bigint)", [
'avatar',
sizeInBytes,
size,
]),
})
.where('id', user.id);
@@ -690,13 +690,13 @@ const upgradeBackgroundImages = async () => {
const dirPathSegment = `${sails.config.custom.backgroundImagesPathSegment}/${dirname}`;
if (backgroundImage) {
const sizeInBytes = await fileManager.getSizeInBytes(
const size = await fileManager.getSize(
`${dirPathSegment}/original.${backgroundImage.extension}`,
);
await knex('background_image')
.update({
size_in_bytes: sizeInBytes,
size_in_bytes: size,
})
.where('id', backgroundImage.id);
} else {
@@ -777,12 +777,10 @@ const upgradeFileAttachments = async () => {
'id',
);
const sizeInBytes = await fileManager.getSizeInBytes(
`${dirPathSegment}/${attachment.data.filename}`,
);
const size = await fileManager.getSize(`${dirPathSegment}/${attachment.data.filename}`);
let encoding = null;
if (sizeInBytes && sizeInBytes <= MAX_SIZE_IN_BYTES_TO_GET_ENCODING) {
if (size && size <= MAX_SIZE_TO_GET_ENCODING) {
const readStream = await fileManager.read(
`${dirPathSegment}/${attachment.data.filename}`,
);
@@ -795,7 +793,7 @@ const upgradeFileAttachments = async () => {
.update({
data: trx.raw(
"?? || jsonb_build_object('fileReferenceId', ?::text, 'sizeInBytes', ?::bigint, 'encoding', ?::text)",
['data', id, sizeInBytes, encoding],
['data', id, size, encoding],
),
})
.where('id', attachment.id);