diff --git a/src/api/client/directory-api.ts b/src/api/client/directory-api.ts index 7a9654d..7078518 100644 --- a/src/api/client/directory-api.ts +++ b/src/api/client/directory-api.ts @@ -1,5 +1,6 @@ import { DirectoryIsNotEmptyError } from 'src/errors/path'; -import { TGFSDirectory } from 'src/model/directory'; +import { FileOrDirectoryDoesNotExistError } from 'src/errors/path'; +import { TGFSDirectory, TGFSFileRef } from 'src/model/directory'; import { validateName } from 'src/utils/validate-name'; import { MetaDataApi } from './metadata-api'; @@ -24,11 +25,22 @@ export class DirectoryApi extends MetaDataApi { return newDirectory; } + public ls( + dir: TGFSDirectory, + fileName?: string, + ): TGFSFileRef | Array { + if (fileName) { + const file = dir.findFile(fileName); + if (file) { + return file; + } + throw new FileOrDirectoryDoesNotExistError(fileName, 'list'); + } + return [...dir.findDirs(), ...dir.findFiles()]; + } + public async deleteEmptyDirectory(directory: TGFSDirectory) { - if ( - directory.findChildren().length > 0 || - directory.findFiles().length > 0 - ) { + if (directory.findDirs().length > 0 || directory.findFiles().length > 0) { throw new DirectoryIsNotEmptyError(); } await this.dangerouslyDeleteDirectory(directory); diff --git a/src/api/client/file-desc-api.ts b/src/api/client/file-desc-api.ts index 4f2921c..8ae0981 100644 --- a/src/api/client/file-desc-api.ts +++ b/src/api/client/file-desc-api.ts @@ -1,6 +1,6 @@ import { MessageNotFound } from 'src/errors/telegram'; import { TGFSFileRef } from 'src/model/directory'; -import { TGFSFile } from 'src/model/file'; +import { TGFSFile, TGFSFileVersion } from 'src/model/file'; import { Logger } from 'src/utils/logger'; import { MessageApi } from './message-api'; @@ -34,22 +34,19 @@ export class FileDescApi extends MessageApi { } public async createFileDesc(fileMsg: GeneralFileMessage): Promise { - const tgfsFile = new TGFSFile(fileMsg.name); + const fd = new TGFSFile(fileMsg.name); if ('empty' in fileMsg) { - tgfsFile.addEmptyVersion(); + fd.addEmptyVersion(); } else { - const id = await this.sendFile(fileMsg); - tgfsFile.addVersionFromFileMessageId(id); + const sentFileMsg = await this.sendFile(fileMsg); + fd.addVersionFromSentFileMessage(sentFileMsg); } - return await this.sendFileDesc(tgfsFile); + return await this.sendFileDesc(fd); } - public async getFileDesc( - fileRef: TGFSFileRef, - withVersionInfo: boolean = true, - ): Promise { + public async getFileDesc(fileRef: TGFSFileRef): Promise { const message = (await this.getMessages([fileRef.getMessageId()]))[0]; if (!message) { @@ -63,25 +60,31 @@ export class FileDescApi extends MessageApi { const fileDesc = TGFSFile.fromObject(JSON.parse(message.text)); - if (withVersionInfo) { - const versions = Object.values(fileDesc.versions); + const versions = Object.values(fileDesc.versions); - const nonEmptyVersions = versions.filter( - (version) => version.messageId > 0, - ); // may contain empty versions + const nonEmptyVersions = versions.filter( + (version) => version.messageId != TGFSFileVersion.EMPTY_FILE, + ); // may contain empty versions - const fileMessages = await this.getMessages( - nonEmptyVersions.map((version) => version.messageId), - ); + const versionsWithoutSizeInfo = nonEmptyVersions.filter( + (version) => version.size == TGFSFileVersion.INVALID_FILE_SIZE, + ); + + const fileMessages = await this.getMessages( + versionsWithoutSizeInfo.map((version) => version.messageId), + ); + + versionsWithoutSizeInfo.forEach((version, i) => { + const fileMessage = fileMessages[i]; + if (fileMessage) { + version.size = Number(fileMessage.document.size); + } else { + version.setInvalid(); + } + }); - nonEmptyVersions.forEach((version, i) => { - const fileMessage = fileMessages[i]; - if (fileMessage) { - version.size = Number(fileMessage.document.size); - } else { - version.setInvalid(); - } - }); + if (versionsWithoutSizeInfo.length > 0) { + await this.updateFileDesc(fileRef.getMessageId(), fileDesc); } return fileDesc; @@ -91,17 +94,27 @@ export class FileDescApi extends MessageApi { fr: TGFSFileRef, fileMsg: GeneralFileMessage, ): Promise { - const fd = await this.getFileDesc(fr, false); + const fd = await this.getFileDesc(fr); if (isFileMessageEmpty(fileMsg)) { fd.addEmptyVersion(); } else { - const messageId = await this.sendFile(fileMsg); - fd.addVersionFromFileMessageId(messageId); + const sentFileMsg = await this.sendFile(fileMsg); + fd.addVersionFromSentFileMessage(sentFileMsg); } return await this.sendFileDesc(fd, fr.getMessageId()); } + public async updateFileDesc( + messageId: number, + fileDesc: TGFSFile, + ): Promise { + return await this.editMessageText( + messageId, + JSON.stringify(fileDesc.toObject()), + ); + } + public async updateFileVersion( fr: TGFSFileRef, fileMsg: GeneralFileMessage, @@ -113,9 +126,10 @@ export class FileDescApi extends MessageApi { fv.setInvalid(); fd.updateVersion(fv); } else { - const messageId = await this.sendFile(fileMsg); + const sentFileMsg = await this.sendFile(fileMsg); const fv = fd.getVersion(versionId); - fv.messageId = messageId; + fv.messageId = sentFileMsg.messageId; + fv.size = sentFileMsg.size.toJSNumber(); fd.updateVersion(fv); } return await this.sendFileDesc(fd, fr.getMessageId()); @@ -125,7 +139,7 @@ export class FileDescApi extends MessageApi { fr: TGFSFileRef, versionId: string, ): Promise { - const fd = await this.getFileDesc(fr, false); + const fd = await this.getFileDesc(fr); fd.deleteVersion(versionId); return await this.sendFileDesc(fd, fr.getMessageId()); } diff --git a/src/api/client/message-api/api.ts b/src/api/client/message-api/api.ts index 24c2b1b..cfde4e1 100644 --- a/src/api/client/message-api/api.ts +++ b/src/api/client/message-api/api.ts @@ -1,14 +1,18 @@ import { Hash, createHash } from 'crypto'; import fs from 'fs'; +import { RPCError } from 'telegram/errors'; + import bigInt from 'big-integer'; import { IBot, TDLibApi } from 'src/api/interface'; +import { SentFileMessage } from 'src/api/types'; import { config } from 'src/config'; import { TechnicalError } from 'src/errors/base'; import { MessageNotFound } from 'src/errors/telegram'; import { TGFSFileVersion } from 'src/model/file'; import { manager } from 'src/server/manager'; +import { flowControl } from 'src/utils/flow-control'; import { Logger } from 'src/utils/logger'; import { getUploader } from './file-uploader'; @@ -29,6 +33,7 @@ export class MessageApi extends MessageBroker { super(tdlib); } + @flowControl() protected async sendText(message: string): Promise { return ( await this.tdlib.bot.sendText({ @@ -38,6 +43,7 @@ export class MessageApi extends MessageBroker { ).messageId; } + @flowControl() protected async editMessageText( messageId: number, message: string, @@ -51,14 +57,19 @@ export class MessageApi extends MessageBroker { }) ).messageId; } catch (err) { - if (err.message === 'message to edit not found') { - throw new MessageNotFound(messageId); - } else { - throw err; + if (err instanceof RPCError) { + if (err.message === 'message to edit not found') { + throw new MessageNotFound(messageId); + } + if (err.errorMessage === 'MESSAGE_NOT_MODIFIED') { + return messageId; + } } + throw err; } } + @flowControl() protected async editMessageMedia( messageId: number, buffer: Buffer, @@ -128,7 +139,9 @@ export class MessageApi extends MessageBroker { // Logger.info(`${(uploaded / totalSize) * 100}% uploaded`); } - private async _sendFile(fileMsg: GeneralFileMessage): Promise { + private async _sendFile( + fileMsg: GeneralFileMessage, + ): Promise { let messageId = TGFSFileVersion.EMPTY_FILE; const uploader = getUploader(this.tdlib, fileMsg, async () => { messageId = ( @@ -138,13 +151,23 @@ export class MessageApi extends MessageBroker { ) ).messageId; }); - await uploader.upload(fileMsg, MessageApi.report, fileMsg.name); + const size = await uploader.upload( + fileMsg, + MessageApi.report, + fileMsg.name, + ); Logger.debug('File sent', JSON.stringify(fileMsg)); - return messageId; + + return { + messageId, + size, + }; } - protected async sendFile(fileMsg: GeneralFileMessage): Promise { + protected async sendFile( + fileMsg: GeneralFileMessage, + ): Promise { if ('stream' in fileMsg) { // Unable to calculate sha256 for file as a stream. So just send it. Logger.debug( @@ -159,17 +182,21 @@ export class MessageApi extends MessageBroker { fileMsg.tags = { sha256: fileHash }; - const existingFile = await this.tdlib.account.searchMessages({ + const existingFileMsg = await this.tdlib.account.searchMessages({ chatId: this.privateChannelId, search: `#sha256IS${fileHash}`, }); - if (existingFile.length > 0) { + if (existingFileMsg.length > 0) { + const msg = existingFileMsg[0]; Logger.debug( `Found file with the same sha256 ${fileHash}, skip uploading`, - JSON.stringify(existingFile[0]), + JSON.stringify(msg), ); - return existingFile[0].messageId; + return { + messageId: msg.messageId, + size: msg.document.size, + }; } return await this._sendFile(fileMsg); diff --git a/src/api/client/message-api/file-uploader.ts b/src/api/client/message-api/file-uploader.ts index 79898a9..fc86006 100644 --- a/src/api/client/message-api/file-uploader.ts +++ b/src/api/client/message-api/file-uploader.ts @@ -8,11 +8,12 @@ import path from 'path'; import { ITDLibClient, TDLibApi } from 'src/api/interface'; import { SendMessageResp, UploadedFile } from 'src/api/types'; -import { Queue, generateFileId, getAppropriatedPartSize } from 'src/api/utils'; +import { generateFileId, getAppropriatedPartSize } from 'src/api/utils'; import { AggregatedError } from 'src/errors/base'; import { FileTooBig } from 'src/errors/telegram'; import { manager } from 'src/server/manager'; import { Logger } from 'src/utils/logger'; +import { Queue } from 'src/utils/queue'; import { FileMessageFromBuffer, @@ -153,7 +154,7 @@ export abstract class FileUploader { totalSize: bigInt.BigInteger, ) => void, fileName?: string, - ): Promise { + ): Promise { const task = manager.createUploadTask(this.fileName, this.fileSize); this.prepare(file); try { @@ -200,6 +201,8 @@ export abstract class FileUploader { task.errors = this.errors; task.complete(); + + return this.fileSize; } } diff --git a/src/api/client/message-api/types.ts b/src/api/client/message-api/types.ts index 5157cd2..7963666 100644 --- a/src/api/client/message-api/types.ts +++ b/src/api/client/message-api/types.ts @@ -1,5 +1,7 @@ import { Readable } from 'stream'; +import { Message } from 'src/api/types'; + export type FileTags = { sha256: string; }; diff --git a/src/api/client/metadata-api.ts b/src/api/client/metadata-api.ts index eccdd15..ca53ab3 100644 --- a/src/api/client/metadata-api.ts +++ b/src/api/client/metadata-api.ts @@ -6,6 +6,7 @@ import { FileDescApi } from './file-desc-api'; export class MetaDataApi extends FileDescApi { private metadata: TGFSMetadata; + private updateMetaDataTimeout: NodeJS.Timeout = null; protected async initMetadata() { this.metadata = await this.getMetadata(); @@ -43,7 +44,6 @@ export class MetaDataApi extends FileDescApi { protected async syncMetadata() { this.metadata.syncWith(await this.getMetadata()); - await this.updateMetadata(); } @@ -57,9 +57,25 @@ export class MetaDataApi extends FileDescApi { 'metadata.json', '', ); + // return new Promise((resolve, reject) => { + // if (this.updateMetaDataTimeout) { + // clearTimeout(this.updateMetaDataTimeout); + // } + // this.updateMetaDataTimeout = setTimeout(async () => { + // try { + + // resolve(undefined); + // } catch (err) { + // reject(err); + // } + // }, 1000); + // }); } else { // doesn't exist, create new metadata and pin - const messageId = await this.sendFile({ buffer, name: 'metadata.json' }); + const { messageId } = await this.sendFile({ + buffer, + name: 'metadata.json', + }); this.metadata.msgId = messageId; await this.pinMessage(messageId); } diff --git a/src/api/ops/copy-dir.ts b/src/api/ops/copy-dir.ts index 85a3e2f..acc90e1 100644 --- a/src/api/ops/copy-dir.ts +++ b/src/api/ops/copy-dir.ts @@ -7,35 +7,35 @@ import { splitPath } from './utils'; export const copyDir = (client: Client) => - async ( - pathFrom: string, - pathTo: string, - ): Promise<{ - from: TGFSDirectory; - to: TGFSDirectory; - }> => { - const [basePathFrom, nameFrom] = splitPath(pathFrom); - if (nameFrom === '') { - return; - } + async ( + pathFrom: string, + pathTo: string, + ): Promise<{ + from: TGFSDirectory; + to: TGFSDirectory; + }> => { + const [basePathFrom, nameFrom] = splitPath(pathFrom); + if (nameFrom === '') { + return; + } - const dir = navigateToDir(client)(basePathFrom); - const dirToCopy = dir.findChildren([nameFrom])[0]; + const dir = navigateToDir(client)(basePathFrom); + const dirToCopy = dir.findDirs([nameFrom])[0]; - if (!dirToCopy) { - throw new FileOrDirectoryDoesNotExistError( - pathFrom, - `move directory from ${pathFrom} to ${pathTo}`, - ); - } + if (!dirToCopy) { + throw new FileOrDirectoryDoesNotExistError( + pathFrom, + `move directory from ${pathFrom} to ${pathTo}`, + ); + } - const [basePathTo, nameTo] = splitPath(pathTo); - const dir2 = navigateToDir(client)(basePathTo); + const [basePathTo, nameTo] = splitPath(pathTo); + const dir2 = navigateToDir(client)(basePathTo); - const res = await client.createDirectory( - { name: nameTo ?? nameFrom, under: dir2 }, - dirToCopy, - ); + const res = await client.createDirectory( + { name: nameTo ?? nameFrom, under: dir2 }, + dirToCopy, + ); - return { from: dirToCopy, to: res }; - }; + return { from: dirToCopy, to: res }; + }; diff --git a/src/api/ops/create-dir.ts b/src/api/ops/create-dir.ts index 06d587a..cd72431 100644 --- a/src/api/ops/create-dir.ts +++ b/src/api/ops/create-dir.ts @@ -20,7 +20,7 @@ export const createDir = const paths = path.split('/').filter((p) => p); let currentDir = client.getRootDirectory(); for (const p of paths) { - const children = currentDir.findChildren([p]); + const children = currentDir.findDirs([p]); if (children.length > 0) { currentDir = children[0]; continue; diff --git a/src/api/ops/list.ts b/src/api/ops/list.ts index 3cd12de..bc4d6fa 100644 --- a/src/api/ops/list.ts +++ b/src/api/ops/list.ts @@ -1,7 +1,6 @@ import { PathLike } from 'fs'; import { Client } from 'src/api'; -import { FileOrDirectoryDoesNotExistError } from 'src/errors/path'; import { TGFSDirectory, TGFSFileRef } from 'src/model/directory'; import { navigateToDir } from './navigate-to-dir'; @@ -9,26 +8,21 @@ import { splitPath } from './utils'; export const list = (client: Client) => - async ( - path: PathLike, - ): Promise> => { - const [basePath, name] = splitPath(path); - const dir = navigateToDir(client)(basePath); + async ( + path: PathLike, + ): Promise> => { + const [basePath, name] = splitPath(path); + const dir = navigateToDir(client)(basePath); - let nextDir = dir; + let nextDir = dir; - if (name) { - nextDir = dir.findChildren([name])[0]; - } - if (nextDir) { - return [...nextDir.findChildren(), ...nextDir.findFiles()]; - } else { - const nextFile = dir.findFiles([name])[0]; - if (nextFile) { - return nextFile; + if (name) { + nextDir = dir.findDir(name); + } + if (nextDir) { + return client.ls(nextDir); } else { - const pathStr = path.toString(); - throw new FileOrDirectoryDoesNotExistError(pathStr, `list ${pathStr}`); + // cannot find a sub-directory with the given name, so assume it's a file + return client.ls(dir, name); } - } - }; + }; diff --git a/src/api/ops/navigate-to-dir.ts b/src/api/ops/navigate-to-dir.ts index b3b36cc..5041fc8 100644 --- a/src/api/ops/navigate-to-dir.ts +++ b/src/api/ops/navigate-to-dir.ts @@ -10,7 +10,7 @@ export const navigateToDir = (client: Client) => (path: string) => { let currentDirectory = client.getRootDirectory(); for (const pathPart of pathParts) { - const directory = currentDirectory.findChildren([pathPart])[0]; + const directory = currentDirectory.findDirs([pathPart])[0]; if (!directory) { throw new FileOrDirectoryDoesNotExistError(path, `navigate to ${path}`); } diff --git a/src/api/ops/remove-dir.ts b/src/api/ops/remove-dir.ts index 910ef66..fe31994 100644 --- a/src/api/ops/remove-dir.ts +++ b/src/api/ops/remove-dir.ts @@ -9,14 +9,14 @@ export const removeDir = const [basePath, name] = splitPath(path); const dir = navigateToDir(client)(basePath); if (!recursive) { - const child = dir.findChildren([name])[0]; + const child = dir.findDirs([name])[0]; if (child) { await client.deleteEmptyDirectory(child); } else { throw new FileOrDirectoryDoesNotExistError(path, `remove dir ${path}`); } } else { - const nextDir = name ? dir.findChildren([name])[0] : dir; + const nextDir = name ? dir.findDirs([name])[0] : dir; await client.dangerouslyDeleteDirectory(nextDir); } }; diff --git a/src/api/types.ts b/src/api/types.ts index 01a9abc..4e0bc28 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -4,6 +4,10 @@ export type Message = { messageId: number; }; +export type SentFileMessage = Message & { + size: bigInt.BigInteger; +}; + export type Chat = { chatId: number; }; diff --git a/src/api/utils/index.ts b/src/api/utils/index.ts index 48b06dd..97ff2dd 100644 --- a/src/api/utils/index.ts +++ b/src/api/utils/index.ts @@ -4,8 +4,6 @@ import { generateRandomBytes, readBigIntFromBuffer } from 'telegram/Helpers'; export { getAppropriatedPartSize } from 'telegram/Utils'; -export { Queue } from './queue'; - export const generateFileId = () => { return readBigIntFromBuffer(generateRandomBytes(8), true, true); }; @@ -27,4 +25,3 @@ export const saveToFile = async ( const content = await saveToBuffer(generator); writeFileSync(path, content); }; - diff --git a/src/commands/executor.ts b/src/commands/executor.ts index d9e73c7..0bbd977 100644 --- a/src/commands/executor.ts +++ b/src/commands/executor.ts @@ -1,5 +1,6 @@ import { Client } from 'src/api'; import { UnknownCommandError } from 'src/errors/cmd'; +import { Logger } from 'src/utils/logger'; import { cp } from './cp'; import { ls } from './ls'; @@ -33,6 +34,6 @@ export class Executor { throw new UnknownCommandError(argv._[0]); } - console.log(rsp); + Logger.stdout(rsp); } } diff --git a/src/model/directory.ts b/src/model/directory.ts index 445d450..bf1f79d 100644 --- a/src/model/directory.ts +++ b/src/model/directory.ts @@ -7,7 +7,7 @@ export class TGFSFileRef { private messageId: number, public name: string, private location: TGFSDirectory, - ) {} + ) { } public toObject(): TGFSFileRefSerialized { return { type: 'FR', messageId: this.messageId, name: this.name }; @@ -36,7 +36,7 @@ export class TGFSDirectory { private parent: TGFSDirectory, private children: TGFSDirectory[] = [], private files: TGFSFileRef[] = [], - ) {} + ) { } public toObject(): TGFSDirectorySerialized { const children = []; @@ -60,12 +60,12 @@ export class TGFSDirectory { dir.files = obj.files ? obj.files - .filter((file) => { - return file.name && file.messageId; - }) - .map((file) => { - return new TGFSFileRef(file.messageId, file.name, dir); - }) + .filter((file) => { + return file.name && file.messageId; + }) + .map((file) => { + return new TGFSFileRef(file.messageId, file.name, dir); + }) : []; obj.children.forEach((child) => { @@ -75,7 +75,7 @@ export class TGFSDirectory { } public createDir(name: string, dir?: TGFSDirectory) { - if (this.findChildren([name]).length) { + if (this.findDirs([name]).length) { throw new FileOrDirectoryAlreadyExistsError(name); } const child = dir @@ -85,7 +85,7 @@ export class TGFSDirectory { return child; } - public findChildren(names?: string[]) { + public findDirs(names?: string[]): Array { if (!names) { return this.children; } else { @@ -94,7 +94,11 @@ export class TGFSDirectory { } } - public findFiles(names?: string[]) { + public findDir(name: string): TGFSDirectory { + return this.findDirs([name])[0]; + } + + public findFiles(names?: string[]): Array { if (!names) { return this.files; } else { @@ -103,11 +107,15 @@ export class TGFSDirectory { } } - public createFileRef(name: string, fileMessageId: number) { - const fr = new TGFSFileRef(fileMessageId, name, this); - if (this.findFiles([fr.name])[0]) { - throw new FileOrDirectoryAlreadyExistsError(fr.name); + public findFile(name: string): TGFSFileRef { + return this.findFiles([name])[0]; + } + + public createFileRef(name: string, fileMessageId: number): TGFSFileRef { + if (this.findFile(name)) { + throw new FileOrDirectoryAlreadyExistsError(name); } + const fr = new TGFSFileRef(fileMessageId, name, this); this.files.push(fr); return fr; } diff --git a/src/model/file.ts b/src/model/file.ts index f906e1d..cb0ad1f 100644 --- a/src/model/file.ts +++ b/src/model/file.ts @@ -1,9 +1,12 @@ import { v4 as uuid } from 'uuid'; +import { SentFileMessage } from 'src/api/types'; + import { TGFSFileObject, TGFSFileVersionSerialized } from './message'; export class TGFSFileVersion { static EMPTY_FILE = -1; + static INVALID_FILE_SIZE = -1; id: string; updatedAt: Date; @@ -16,6 +19,7 @@ export class TGFSFileVersion { id: this.id, updatedAt: this.updatedAt.getTime(), messageId: this.messageId, + size: this.size, }; } @@ -27,11 +31,12 @@ export class TGFSFileVersion { return tgfsFileVersion; } - static fromFileMessageId(fileMessageId: number): TGFSFileVersion { + static fromSentFileMessage(msg: SentFileMessage): TGFSFileVersion { const tgfsFileVersion = new TGFSFileVersion(); tgfsFileVersion.id = uuid(); tgfsFileVersion.updatedAt = new Date(); - tgfsFileVersion.messageId = fileMessageId; + tgfsFileVersion.messageId = msg.messageId; + tgfsFileVersion.size = msg.size.toJSNumber(); return tgfsFileVersion; } @@ -42,12 +47,14 @@ export class TGFSFileVersion { tgfsFileVersion.id = tgfsFileVersionObject['id']; tgfsFileVersion.updatedAt = new Date(tgfsFileVersionObject['updatedAt']); tgfsFileVersion.messageId = tgfsFileVersionObject['messageId']; + tgfsFileVersion.size = + tgfsFileVersionObject['size'] ?? TGFSFileVersion.INVALID_FILE_SIZE; return tgfsFileVersion; } setInvalid() { this.messageId = TGFSFileVersion.EMPTY_FILE; - this.size = 0; + this.size = TGFSFileVersion.INVALID_FILE_SIZE; } } @@ -110,8 +117,8 @@ export class TGFSFile { this.latestVersionId = version.id; } - addVersionFromFileMessageId(fileMessageId: number) { - const version = TGFSFileVersion.fromFileMessageId(fileMessageId); + addVersionFromSentFileMessage(msg: SentFileMessage) { + const version = TGFSFileVersion.fromSentFileMessage(msg); this.addVersion(version); this.latestVersionId = version.id; } diff --git a/src/model/message.ts b/src/model/message.ts index 64da902..ac90c88 100644 --- a/src/model/message.ts +++ b/src/model/message.ts @@ -3,6 +3,7 @@ export class TGFSFileVersionSerialized { id: string; updatedAt: number; messageId: number; + size: number; } export class TGFSFileObject { diff --git a/src/server/webdav/index.ts b/src/server/webdav/index.ts index f874761..daa2cbd 100644 --- a/src/server/webdav/index.ts +++ b/src/server/webdav/index.ts @@ -41,7 +41,7 @@ export const webdavServer = ( next(); }); server.afterRequest((ctx, next) => { - Logger.info(ctx.request.method, ctx.response.statusCode); + Logger.info(ctx.request.method, ctx.response.statusCode, ctx.responseBody); next(); }); server.setFileSystemSync('/', new TGFSFileSystem(client)); diff --git a/src/server/webdav/tgfs-filesystem.ts b/src/server/webdav/tgfs-filesystem.ts index 5bbe6fc..d36f145 100644 --- a/src/server/webdav/tgfs-filesystem.ts +++ b/src/server/webdav/tgfs-filesystem.ts @@ -17,9 +17,13 @@ import { OpenReadStreamInfo, OpenWriteStreamInfo, Path, + PropertyAttributes, + PropertyBag, PropertyManagerInfo, ReadDirInfo, + ResourcePropertyValue, ResourceType, + Return2Callback, ReturnCallback, SimpleCallback, SizeInfo, @@ -79,6 +83,8 @@ const call = }); }; +class TGFSPropertyManager extends LocalPropertyManager {} + export class TGFSFileSystem extends FileSystem { constructor(public readonly tgClient: Client) { super(new TGFSSerializer()); @@ -227,7 +233,7 @@ export class TGFSFileSystem extends FileSystem { ctx: PropertyManagerInfo, callback: ReturnCallback, ): void { - callback(null, new LocalPropertyManager()); + callback(null, new TGFSPropertyManager()); } protected _type( diff --git a/src/utils/flow-control.ts b/src/utils/flow-control.ts new file mode 100644 index 0000000..6276c50 --- /dev/null +++ b/src/utils/flow-control.ts @@ -0,0 +1,65 @@ +// a decorator limiting the number of times a function can be called in a second +// reqeusts exceeding limit will be awaiting for the next second +import { Queue } from './queue'; +import { sleep } from './sleep'; + +const queue: Queue<{ + resolve: (value?: unknown) => void; + reject: (reason?: unknown) => void; + fn: Function; + args: any[]; +}> = new Queue(); + +const delay = 50; + +(async () => { + while (true) { + await sleep(delay); + if (queue.isEmpty()) { + continue; + } + + const { resolve, reject, fn, args } = queue.dequeue(); + + try { + const res = await fn(...args); + resolve(res); + } catch (err) { + reject(err); + } + } +})(); + +export function flowControl() { + return function ( + _target: any, + _propertyKey: string, + descriptor: PropertyDescriptor, + ) { + const originalMethod = descriptor.value!; + + descriptor.value = function (...args: any[]) { + return new Promise((resolve, reject) => { + if (process.env['NODE_ENV'] === 'test') { + originalMethod + .bind(this)(...args) + .then((res) => { + resolve(res); + }) + .catch((err) => { + reject(err); + }); + } else { + queue.enqueue({ + resolve, + reject, + fn: originalMethod.bind(this), + args, + }); + } + }); + }; + + return descriptor; + }; +} diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 88cfea3..c864212 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -40,4 +40,8 @@ export class Logger { `[${this.getTime()}] [ERROR] ${this.prefix()} ${this.errorMsg(err)}`, ); } + + static stdout(...args: any[]) { + console.log(...args); + } } diff --git a/src/api/utils/queue.ts b/src/utils/queue.ts similarity index 100% rename from src/api/utils/queue.ts rename to src/utils/queue.ts diff --git a/src/utils/retry.ts b/src/utils/retry.ts index c8d6f1b..ae23c59 100644 --- a/src/utils/retry.ts +++ b/src/utils/retry.ts @@ -2,7 +2,7 @@ import { Logger } from 'src/utils/logger'; export function retry( retries: number = 10, - backoff: number = 500, + backoff: number = 50, ): MethodDecorator { return function ( _target: Object, diff --git a/test/api/model/operations.spec.ts b/test/api/model/operations.spec.ts index 78df0c8..cd0d78c 100644 --- a/test/api/model/operations.spec.ts +++ b/test/api/model/operations.spec.ts @@ -2,13 +2,14 @@ import fs from 'fs'; import { Client } from 'src/api'; import { saveToBuffer, saveToFile } from 'src/api/utils'; +import { Logger } from 'src/utils/logger'; import { sleep } from 'src/utils/sleep'; import { createMockClient } from '../../utils/mock-tg-client'; describe('file and directory operations', () => { beforeAll(() => { - console.info = jest.fn(); + Logger.info = jest.fn(); }); describe('create / remove directories', () => { @@ -21,7 +22,7 @@ describe('file and directory operations', () => { it('should create a directory', async () => { const root = client.getRootDirectory(); const d1 = await client.createDirectory({ name: 'd1', under: root }); - expect(root.findChildren(['d1'])[0]).toEqual(d1); + expect(root.findDirs(['d1'])[0]).toEqual(d1); }); it('should throw an error if the directory name is illegal', async () => { @@ -40,7 +41,7 @@ describe('file and directory operations', () => { const d1 = await client.createDirectory({ name: 'd1', under: root }); await client.dangerouslyDeleteDirectory(d1); - expect(root.findChildren(['d1'])[0]).toBeUndefined(); + expect(root.findDirs(['d1'])[0]).toBeUndefined(); }); it('should remove all directories', async () => { @@ -50,7 +51,7 @@ describe('file and directory operations', () => { }); await client.createDirectory({ name: 'd2', under: d1 }); await client.dangerouslyDeleteDirectory(client.getRootDirectory()); - expect(client.getRootDirectory().findChildren(['d1'])[0]).toBeUndefined(); + expect(client.getRootDirectory().findDirs(['d1'])[0]).toBeUndefined(); }); }); diff --git a/test/cmd/cmd.spec.ts b/test/cmd/cmd.spec.ts index 3f0fecc..d36a538 100644 --- a/test/cmd/cmd.spec.ts +++ b/test/cmd/cmd.spec.ts @@ -7,6 +7,7 @@ import { createDir, list, uploadFromBytes } from 'src/api/ops'; import { Executor } from 'src/commands/executor'; import { parser } from 'src/commands/parser'; import { TGFSDirectory } from 'src/model/directory'; +import { Logger } from 'src/utils/logger'; import { createMockClient } from '../utils/mock-tg-client'; @@ -21,8 +22,8 @@ const getExecutor = (client: Client) => { describe('commands', () => { beforeAll(() => { - console.log = jest.fn(); - console.info = jest.fn(); + Logger.info = jest.fn(); + Logger.stdout = jest.fn(); }); describe('ls', () => { @@ -35,7 +36,7 @@ describe('commands', () => { jest.replaceProperty(process, 'argv', ['ls', '/']); await executor.execute(parse()); - expect(console.log).toHaveBeenCalledWith('d1 f1'); + expect(Logger.stdout).toHaveBeenCalledWith('d1 f1'); }); it('should display file info', async () => { @@ -49,7 +50,7 @@ describe('commands', () => { const fr = client.getRootDirectory().findFiles(['f1'])[0]; const fd = await client.getFileDesc(fr); - expect(console.log).toHaveBeenCalledWith( + expect(Logger.stdout).toHaveBeenCalledWith( expect.stringContaining(fd.latestVersionId), ); }); @@ -176,7 +177,7 @@ describe('commands', () => { jest.replaceProperty(process, 'argv', ['rm', '/d1']); await executor.execute(parse()); - expect(client.getRootDirectory().findChildren(['d1']).length).toEqual(0); + expect(client.getRootDirectory().findDirs(['d1']).length).toEqual(0); }); it('should throw an error if path does not exist', async () => { @@ -184,7 +185,7 @@ describe('commands', () => { const executor = getExecutor(client); jest.replaceProperty(process, 'argv', ['rm', '/not-exist']); - expect(executor.execute(parse())).rejects.toThrowError(); + expect(executor.execute(parse())).rejects.toThrow(); }); it('should throw an error if trying to remove a directory that is not empty', async () => { @@ -214,7 +215,7 @@ describe('commands', () => { jest.replaceProperty(process, 'argv', ['rm', '/d1', '-r']); await executor.execute(parse()); - expect(client.getRootDirectory().findChildren(['d1']).length).toEqual(0); + expect(client.getRootDirectory().findDirs(['d1']).length).toEqual(0); }); }); diff --git a/test/server/webdav.spec.ts b/test/server/webdav.spec.ts index c3534df..147a58d 100644 --- a/test/server/webdav.spec.ts +++ b/test/server/webdav.spec.ts @@ -4,6 +4,7 @@ import supertest from 'supertest'; import { v2 as webdav } from 'webdav-server'; import { TGFSFileSystem } from 'src/server/webdav/tgfs-filesystem'; +import { Logger } from 'src/utils/logger'; import { createMockClient } from '../utils/mock-tg-client'; @@ -19,7 +20,7 @@ describe('TGFSFileSystem', () => { }; beforeAll(async () => { - console.info = jest.fn(); + Logger.info = jest.fn(); }); describe('list directory', () => { diff --git a/test/api/queue.spec.ts b/test/test_utils/queue.spec.ts similarity index 97% rename from test/api/queue.spec.ts rename to test/test_utils/queue.spec.ts index b667eeb..87b4837 100644 --- a/test/api/queue.spec.ts +++ b/test/test_utils/queue.spec.ts @@ -1,4 +1,4 @@ -import { Queue } from 'src/api/utils'; +import { Queue } from 'src/utils/queue'; describe('Queue', () => { let queue: Queue;