liberama/server/core/FileDecompressor.js

const fs = require('fs-extra');
const zlib = require('zlib');
const path = require('path');
const unbzip2Stream = require('unbzip2-stream');
const tar = require('tar-fs');
const iconv = require('iconv-lite');

const ZipStreamer = require('./Zip/ZipStreamer');
const appLogger = new (require('./AppLogger'))();//singleton
const FileDetector = require('./FileDetector');
const textUtils = require('./Reader/BookConverter/textUtils');
const utils = require('./utils');

class FileDecompressor {
    constructor(limitFileSize = 0) {
        this.detector = new FileDetector();
        this.limitFileSize = limitFileSize;

        this.rarPath = '/usr/bin/rar';
        this.rarExists = false;
        (async() => {
            if (await fs.pathExists(this.rarPath))
                this.rarExists = true;
        })();
    }

    async decompressNested(filename, outputDir) {
        await fs.ensureDir(outputDir);
        
        const fileType = await this.detector.detectFile(filename);

        let result = {
            sourceFile: filename,
            sourceFileType: fileType,
            selectedFile: filename,
            filesDir: outputDir,
            files: []
        };

        if (!fileType || !(
                    fileType.ext == 'zip' || fileType.ext == 'bz2' || fileType.ext == 'gz'
                    || fileType.ext == 'tar' || (this.rarExists && fileType.ext == 'rar')
                )
            ) {
            return result;
        }

        result.files = await this.decompressTarZZ(fileType.ext, filename, outputDir);

        let sel = filename;
        let max = 0;
        if (result.files.length) {
            //ищем файл с максимальным размером
            for (let file of result.files) {
                if (file.size > max) {
                    sel = `${outputDir}/${file.path}`;
                    max = file.size;
                }
            }
        }
        result.selectedFile = sel;

        if (sel != filename) {
            result.nesting = await this.decompressNested(sel, `${outputDir}/${utils.randomHexString(10)}`);
        }

        return result;
    }

    async unpack(filename, outputDir) {
        const fileType = await this.detector.detectFile(filename);
        if (!fileType)
            throw new Error('Не удалось определить формат файла');

        return await this.decompress(fileType.ext, filename, outputDir);
    }

    async unpackTarZZ(filename, outputDir) {
        const fileType = await this.detector.detectFile(filename);
        if (!fileType)
            throw new Error('Не удалось определить формат файла');

        return await this.decompressTarZZ(fileType.ext, filename, outputDir);
    }

    async decompressTarZZ(fileExt, filename, outputDir) {
        const files = await this.decompress(fileExt, filename, outputDir);
        if (fileExt == 'tar' || files.length != 1)
            return files;

        const tarFilename = `${outputDir}/${files[0].path}`;
        const fileType = await this.detector.detectFile(tarFilename);
        if (!fileType || fileType.ext != 'tar')
            return files;

        const newTarFilename = `${outputDir}/${utils.randomHexString(30)}`;
        await fs.rename(tarFilename, newTarFilename);
        
        const tarFiles = await this.decompress('tar', newTarFilename, outputDir);
        await fs.remove(newTarFilename);

        return tarFiles;
    }

    async decompress(fileExt, filename, outputDir) {
        let files = [];

        if (fileExt == 'rar' && this.rarExists) {
            files = await this.unRar(filename, outputDir);
            return files;
        }

        switch (fileExt) {
            case 'zip':
                files = await this.unZip(filename, outputDir);
                break;
            case 'bz2':
                files = await this.unBz2(filename, outputDir);
                break;
            case 'gz':
                files = await this.unGz(filename, outputDir);
                break;
            case 'tar':
                files = await this.unTar(filename, outputDir);
                break;
            default:
                throw new Error(`FileDecompressor: неизвестный формат файла '${fileExt}'`);
        }

        return files;
    }

    async unZip(filename, outputDir) {
        const zip = new ZipStreamer();
        try {
            return await zip.unpack(filename, outputDir, {
                limitFileSize: this.limitFileSize, 
                limitFileCount: 10000,
                decodeEntryNameCallback: (nameRaw) => {
                    return utils.bufferRemoveZeroes(nameRaw);
                }
            });
        } catch (e) {
            fs.emptyDir(outputDir);
            return await zip.unpack(filename, outputDir, {
                limitFileSize: this.limitFileSize, 
                limitFileCount: 10000,
                decodeEntryNameCallback: (nameRaw) => {
                    nameRaw = utils.bufferRemoveZeroes(nameRaw);
                    const enc = textUtils.getEncodingLite(nameRaw);
                    if (enc.indexOf('ISO-8859') < 0) {
                        return iconv.decode(nameRaw, enc);
                    }
                    return nameRaw;
                }
            });
        }
    }

    unBz2(filename, outputDir) {
        return this.decompressByStream(unbzip2Stream(), filename, outputDir);
    }

    unGz(filename, outputDir) {
        return this.decompressByStream(zlib.createGunzip(), filename, outputDir);
    }

    unTar(filename, outputDir) {
        return new Promise((resolve, reject) => { (async() => {
            const files = [];

            if (this.limitFileSize) {
                if ((await fs.stat(filename)).size > this.limitFileSize) {
                    reject(new Error('Файл слишком большой'));
                    return;
                }
            }

            const tarExtract = tar.extract(outputDir, {
                map: (header) => {
                    files.push({path: header.name, size: header.size});
                    return header;
                }
            });

            tarExtract.on('finish', () => {
                resolve(files);
            });

            tarExtract.on('error', (err) => {
                reject(err);
            });

            const inputStream = fs.createReadStream(filename);
            inputStream.on('error', (err) => {
                reject(err);
            });

            inputStream.pipe(tarExtract);
        })().catch(reject); });
    }

    decompressByStream(stream, filename, outputDir) {
        return new Promise((resolve, reject) => { (async() => {
            const file = {path: path.parse(filename).name};
            let outFilename = `${outputDir}/${file.path}`;
            if (await fs.pathExists(outFilename)) {
                file.path = `${utils.randomHexString(10)}-${file.path}`;
                outFilename = `${outputDir}/${file.path}`;
            }
        
            const inputStream = fs.createReadStream(filename);
            const outputStream = fs.createWriteStream(outFilename);

            outputStream.on('finish', async() => {
                try {
                    file.size = (await fs.stat(outFilename)).size;
                } catch (e) {
                    reject(e);
                }
                resolve([file]);
            });

            stream.on('error', reject);

            if (this.limitFileSize) {
                let readSize = 0;
                stream.on('data', (buffer) => {
                    readSize += buffer.length;
                    if (readSize > this.limitFileSize)
                        stream.destroy(new Error('Файл слишком большой'));
                });
            }

            inputStream.on('error', reject);
            outputStream.on('error', reject);
        
            inputStream.pipe(stream).pipe(outputStream);
        })().catch(reject); });
    }

    async unRar(filename, outputDir) {
        try {
            const args = ['x', '-p-', '-y', filename, `${outputDir}`];
            const result = await utils.spawnProcess(this.rarPath, {
                killAfter: 60,
                args
            });

            if (result.code == 0) {
                const files = [];
                await utils.findFiles(async(file) => {
                    const stat = await fs.stat(file);
                    files.push({path: path.relative(outputDir, file), size: stat.size});
                }, outputDir);

                return files;

            } else {
                const error = `${result.code}|FORLOG|, exec: ${this.rarPath}, args: ${args.join(' ')}, stdout: ${result.stdout}, stderr: ${result.stderr}`;
                throw new Error(`Архиватор Rar завершился с ошибкой: ${error}`);
            }
        } catch(e) {
            if (e.status == 'killed') {
                throw new Error('Слишком долгое ожидание архиватора Rar');
            } else if (e.status == 'error') {
                throw new Error(e.error);
            } else {
                throw new Error(e);
            }
        }
    }

    async gzipBuffer(buf) {
        return new Promise((resolve, reject) => {
            zlib.gzip(buf, {level: 1}, (err, result) => {
                if (err) reject(err);
                resolve(result);
            });
        });
    }

    async gzipFile(inputFile, outputFile, level = 1) {
        return new Promise((resolve, reject) => {
            const gzip = zlib.createGzip({level});
            const input = fs.createReadStream(inputFile);
            const output = fs.createWriteStream(outputFile);

            input.pipe(gzip).pipe(output).on('finish', (err) => {
                if (err) reject(err);
                else resolve();
            });
        });
    }

    async gzipFileIfNotExists(filename, outDir, isMaxCompression) {
        const hash = await utils.getFileHash(filename, 'sha256', 'hex');

        const outFilename = `${outDir}/${hash}`;

        if (!await fs.pathExists(outFilename)) {
            await this.gzipFile(filename, outFilename, (isMaxCompression ? 9 : 1));

            // переупакуем через некоторое время на максималках, если упаковали плохо
            if (!isMaxCompression) {
                const filenameCopy = `${filename}.copy`;
                await fs.copy(filename, filenameCopy);

                (async() => {
                    await utils.sleep(5000);
                    const filenameGZ = `${filename}.gz`;
                    await this.gzipFile(filenameCopy, filenameGZ, 9);

                    await fs.move(filenameGZ, outFilename, {overwrite: true});

                    await fs.remove(filenameCopy);
                })().catch((e) => { if (appLogger.inited) appLogger.log(LM_ERR, `FileDecompressor.gzipFileIfNotExists: ${e.message}`) });
            }
        } else {
            await utils.touchFile(outFilename);
        }

        return outFilename;
    }
}

module.exports = FileDecompressor;