Merge branch 'release/0.9.10'

client/api/reader.js

@@ -181,9 +181,8 @@ class Reader {
             maxUploadFileSize = 10*1024*1024;
         if (file.size > maxUploadFileSize)
             throw new Error(`Размер файла превышает ${maxUploadFileSize} байт`);
-
         let formData = new FormData();
-        formData.append('file', file);
+        formData.append('file', file, file.name);
 
         const options = {
             headers: {

client/components/App.vue

@@ -128,6 +128,10 @@ class App extends Vue {
 
         this.setAppTitle();
         (async() => {
+            //запросим persistent storage
+            if (navigator.storage && navigator.storage.persist) {
+                navigator.storage.persist();
+            }
             await this.routerReady();
             this.redirectIfNeeded();
         })();

client/components/Reader/LoaderPage/LoaderPage.vue

@@ -7,8 +7,8 @@
             <span class="greeting"><b>{{ title }}</b></span>
             <div class="q-my-sm"></div>
             <span class="greeting">Добро пожаловать!</span>
-            <span class="greeting">Поддерживаются форматы: <b>fb2, html, txt</b> и сжатие: <b>zip, bz2, gz</b></span>
-            <span v-if="isExternalConverter" class="greeting">...а также форматы: <b>rtf, doc, docx, pdf, epub, mobi</b></span>
+            <span class="greeting">Поддерживаются форматы: <b>fb2, html, txt</b> и сжатие: <b>zip, bz2, gz<span v-if="isExternalConverter">, rar</span></b></span>
+            <span v-if="isExternalConverter" class="greeting">...а также частично форматы: <b>epub, mobi, rtf, doc, docx, pdf, djvu</b></span>
         </div>
 
         <div class="col-auto column justify-start items-center no-wrap overflow-hidden">

client/components/Reader/Reader.vue

@@ -846,8 +846,12 @@ class Reader extends Vue {
 
         let url = encodeURI(decodeURI(opts.url));
 
+        //TODO: убрать конвертирование 'file://' после 06.2021
+        if (url.length == 71 && url.indexOf('file://') == 0)
+            url = url.replace(/^file/, 'disk');
+
         if ((url.indexOf('http://') != 0) && (url.indexOf('https://') != 0) &&
-            (url.indexOf('file://') != 0))
+            (url.indexOf('disk://') != 0))
             url = 'http://' + url;
 
         // уже просматривается сейчас
@@ -924,7 +928,8 @@ class Reader extends Vue {
                         url,
                         skipCheck: (opts.skipCheck ? true : false),
                         isText: (opts.isText ? true : false),
-                        enableSitesFilter: this.enableSitesFilter
+                        enableSitesFilter: this.enableSitesFilter,
+                        uploadFileName: (opts.uploadFileName ? opts.uploadFileName : ''),
                     },
                     (state) => {
                         progress.setState(state);
@@ -977,7 +982,7 @@ class Reader extends Vue {
 
             progress.hide(); this.progressActive = false;
 
-            await this.loadBook({url});
+            await this.loadBook({url, uploadFileName: opts.file.name});
         } catch (e) {
             progress.hide(); this.progressActive = false;
             this.loaderActive = true;

client/components/Reader/RecentBooksPage/RecentBooksPage.vue

@@ -296,7 +296,7 @@ class RecentBooksPage extends Vue {
 
     isUrl(url) {
         if (url)
-            return (url.indexOf('file://') != 0);
+            return (url.indexOf('disk://') != 0);
         else
             return false;
     }

client/components/Reader/TextPage/DrawHelper.js

@@ -159,7 +159,7 @@ export default class DrawHelper {
         const fh = h - 2*pad;
         const fh2 = fh/2;
 
-        const t1 = `${Math.floor((bookPos + 1)/1000)}k/${Math.floor(textLength/1000)}k`;
+        const t1 = `${Math.floor((bookPos + 1)/1000)}/${Math.floor(textLength/1000)}`;
         const w1 = this.measureTextFont(t1, font) + fh2;
         const read = (bookPos + 1)/textLength;
         const t2 = `${(read*100).toFixed(2)}%`;

client/components/Reader/TextPage/TextPage.vue

@@ -1053,7 +1053,7 @@ class TextPage extends Vue {
 
     onStatusBarClick() {
         const url = this.meta.url;
-        if (url && url.indexOf('file://') != 0) {
+        if (url && url.indexOf('disk://') != 0) {
             window.open(url, '_blank');
         } else {
             this.$root.stdDialog.alert('Оригинал недоступен, т.к. файл книги был загружен с локального диска.', ' ', {color: 'info'});

client/components/Reader/share/bookManager.js

@@ -4,7 +4,7 @@ import _ from 'lodash';
 import * as utils from '../../../share/utils';
 import BookParser from './BookParser';
 
-const maxDataSize = 300*1024*1024;//compressed bytes
+const maxDataSize = 500*1024*1024;//compressed bytes
 
 //локальный кэш метаданных книг, ограничение maxDataSize
 const bmMetaStore = localForage.createInstance({
@@ -64,8 +64,12 @@ class BookManager {
 
             await this.cleanRecentBooks();
 
-            if (this.recentRev > 10)
-                await bmRecentStoreOld.clear();
+            //TODO: убрать после 06.2021, когда bmRecentStoreOld устареет
+            {
+                await this.convertFileToDiskPrefix();
+                if (this.recentRev > 10)
+                    await bmRecentStoreOld.clear();
+            }
         } else {//TODO: убрать после 06.2021, когда bmRecentStoreOld устареет
             this.recentLast = await bmRecentStoreOld.getItem('recent-last');
             if (this.recentLast) {
@@ -455,6 +459,33 @@ class BookManager {
         return isDel;
     }
 
+    async convertFileToDiskPrefix() {
+        let isConverted = false;
+
+        const newRecent = {};
+        for (let key of Object.keys(this.recent)) {
+            let newKey = key;
+            let newUrl = this.recent[key].url;
+
+            if (newKey.indexOf('66696c65') == 0) {
+                newKey = newKey.replace(/^66696c65/, '6469736b');
+                if (newUrl)
+                    newUrl = newUrl.replace(/^file/, 'disk');
+                isConverted = true;
+            }
+
+            newRecent[newKey] = this.recent[key];
+            newRecent[newKey].key = newKey;
+            if (newUrl)
+                newRecent[newKey].url = newUrl;
+        }
+        if (isConverted) {
+            this.recent = newRecent;
+            await this.recentSetItem(null, true);
+        }
+        return isConverted;
+    }
+
     mostRecentBook() {
         if (this.recentLastKey) {
             return this.recent[this.recentLastKey];

client/components/Reader/versionHistory.js

@@ -1,4 +1,17 @@
 export const versionHistory = [
+{
+    showUntil: '2020-12-10',
+    header: '0.9.10 (2020-12-03)',
+    content:
+`
+<ul>
+    <li>добавлена частичная поддержка формата Djvu</li>
+    <li>добавлена поддержка Rar-архивов</li>
+    <li>исправления багов</li>
+</ul>
+`
+},
+
 {
     showUntil: '2020-11-20',
     header: '0.9.9 (2020-11-21)',

client/components/share/Dialog.vue

@@ -1,5 +1,5 @@
 <template>
-    <q-dialog v-model="active">
+    <q-dialog v-model="active" no-route-dismiss>
         <div class="column bg-white no-wrap">
             <div class="header row">
                 <div class="caption col row items-center q-ml-md">

client/components/share/StdDialog.vue

@@ -1,5 +1,5 @@
 <template>
-    <q-dialog ref="dialog" v-model="active" @show="onShow" @hide="onHide">
+    <q-dialog ref="dialog" v-model="active" @show="onShow" @hide="onHide" no-route-dismiss>
         <slot></slot>
 
         <!--------------------------------------------------->

docs/omnireader.ru/README.md

@@ -34,8 +34,10 @@ sudo -u www-data tar xvf calibre-5.5.0-x86_64.txz -C /home/liberama/data/calibre
 
 ### external converters
 ```
+sudo apt install rar
 sudo apt install libreoffice
 sudo apt install poppler-utils
+sudo apt install djvulibre-bin
 ```
 
 ### nginx, server config

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "Liberama",
-  "version": "0.9.9",
+  "version": "0.9.10",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -10124,6 +10124,21 @@
       "integrity": "sha512-ISBaA8xQNmwELC7eOjqFKMESB2VIqt4PPDD0nsS95b/9dZXvVKOlz9keMSnoGGKcOHXfTvDD6WMaRoSc9UuhRA==",
       "dev": true
     },
+    "pidusage": {
+      "version": "2.0.21",
+      "resolved": "https://registry.npmjs.org/pidusage/-/pidusage-2.0.21.tgz",
+      "integrity": "sha512-cv3xAQos+pugVX+BfXpHsbyz/dLzX+lr44zNMsYiGxUw+kV5sgQCIcLd1z+0vq+KyC7dJ+/ts2PsfgWfSC3WXA==",
+      "requires": {
+        "safe-buffer": "^5.2.1"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+        }
+      }
+    },
     "pify": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "Liberama",
-  "version": "0.9.9",
+  "version": "0.9.10",
   "author": "Book Pauk <bookpauk@gmail.com>",
   "license": "CC0-1.0",
   "repository": "bookpauk/liberama",
@@ -8,7 +8,7 @@
     "node": ">=10.0.0"
   },
   "scripts": {
-    "dev": "nodemon --inspect --exec 'node server'",
+    "dev": "nodemon --inspect --ignore server/public --ignore server/data --exec 'node server'",
     "build:client": "webpack --config build/webpack.prod.config.js",
     "build:linux": "npm run build:client && node build/linux && pkg -t latest-linux-x64 -o dist/linux/liberama .",
     "build:win": "npm run build:client && node build/win && pkg -t latest-win-x64 -o dist/win/liberama .",
@@ -72,6 +72,7 @@
     "multer": "^1.4.2",
     "pako": "^1.0.11",
     "path-browserify": "^1.0.0",
+    "pidusage": "^2.0.21",
     "quasar": "^1.14.3",
     "safe-buffer": "^5.2.0",
     "sjcl": "^1.0.8",

server/controllers/ReaderController.js

@@ -22,6 +22,7 @@ class ReaderController extends BaseController {
                 enableSitesFilter: (request.hasOwnProperty('enableSitesFilter') ? request.enableSitesFilter : true),
                 skipCheck: (request.hasOwnProperty('skipCheck') ? request.skipCheck : false),
                 isText: (request.hasOwnProperty('isText') ? request.isText : false),
+                uploadFileName: (request.hasOwnProperty('uploadFileName') ? request.uploadFileName : false),
             });
             const state = this.workerState.getState(workerId);
             return (state ? state : {});

server/controllers/WebSocketController.js

@@ -138,6 +138,7 @@ class WebSocketController {
             i++;
             if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера
                 this.send({state: 'error', error: 'Время ожидания процесса истекло'}, req, ws);
+                break;
             }
             i = (prevProgress != state.progress || prevState != state.state ? 1 : i);
         }        

server/core/FileDecompressor.js

@@ -15,6 +15,13 @@ class FileDecompressor {
     constructor(limitFileSize = 0) {
         this.detector = new FileDetector();
         this.limitFileSize = limitFileSize;
+
+        this.rarPath = '/usr/bin/rar';
+        this.rarExists = false;
+        (async() => {
+            if (await fs.pathExists(this.rarPath))
+                this.rarExists = true;
+        })();
     }
 
     async decompressNested(filename, outputDir) {
@@ -30,7 +37,11 @@ class FileDecompressor {
             files: []
         };
 
-        if (!fileType || !(fileType.ext == 'zip' || fileType.ext == 'bz2' || fileType.ext == 'gz' || fileType.ext == 'tar')) {
+        if (!fileType || !(
+                    fileType.ext == 'zip' || fileType.ext == 'bz2' || fileType.ext == 'gz'
+                    || fileType.ext == 'tar' || (this.rarExists && fileType.ext == 'rar')
+                )
+            ) {
             return result;
         }
 
@@ -94,6 +105,11 @@ class FileDecompressor {
     async decompress(fileExt, filename, outputDir) {
         let files = [];
 
+        if (fileExt == 'rar' && this.rarExists) {
+            files = await this.unRar(filename, outputDir);
+            return files;
+        }
+
         switch (fileExt) {
             case 'zip':
                 files = await this.unZip(filename, outputDir);
@@ -123,8 +139,7 @@ class FileDecompressor {
                 decodeEntryNameCallback: (nameRaw) => {
                     return utils.bufferRemoveZeroes(nameRaw);
                 }
-            }
-);
+            });
         } catch (e) {
             fs.emptyDir(outputDir);
             return await zip.unpack(filename, outputDir, {
@@ -222,7 +237,39 @@ class FileDecompressor {
         
             inputStream.pipe(stream).pipe(outputStream);
         })().catch(reject); });
-   }
+    }
+
+    async unRar(filename, outputDir) {
+        try {
+            const args = ['x', '-p-', '-y', filename, `${outputDir}`];
+            const result = await utils.spawnProcess(this.rarPath, {
+                killAfter: 60,
+                args
+            });
+
+            if (result.code == 0) {
+                const files = [];
+                await utils.findFiles(async(file) => {
+                    const stat = await fs.stat(file);
+                    files.push({path: path.relative(outputDir, file), size: stat.size});
+                }, outputDir);
+
+                return files;
+
+            } else {
+                const error = `${result.code}|FORLOG|, exec: ${this.rarPath}, args: ${args.join(' ')}, stdout: ${result.stdout}, stderr: ${result.stderr}`;
+                throw new Error(`Архиватор Rar завершился с ошибкой: ${error}`);
+            }
+        } catch(e) {
+            if (e.status == 'killed') {
+                throw new Error('Слишком долгое ожидание архиватора Rar');
+            } else if (e.status == 'error') {
+                throw new Error(e.error);
+            } else {
+                throw new Error(e);
+            }
+        }
+    }
 
     async gzipBuffer(buf) {
         return new Promise((resolve, reject) => {

server/core/Reader/BookConverter/.gitignore

@@ -1 +0,0 @@
-test

server/core/Reader/BookConverter/ConvertBase.js

@@ -6,7 +6,7 @@ const LimitedQueue = require('../../LimitedQueue');
 const textUtils = require('./textUtils');
 const utils = require('../../utils');
 
-const queue = new LimitedQueue(2, 20, 3*60*1000);//3 минуты ожидание подвижек
+const queue = new LimitedQueue(3, 20, 3*60*1000);//3 минуты ожидание подвижек
 
 class ConvertBase {
     constructor(config) {
@@ -15,6 +15,7 @@ class ConvertBase {
         this.calibrePath = `${config.dataDir}/calibre/ebook-convert`;
         this.sofficePath = '/usr/bin/soffice';
         this.pdfToHtmlPath = '/usr/bin/pdftohtml';
+        this.ddjvuPath = '/usr/bin/ddjvu';
     }
 
     async run(data, opts) {// eslint-disable-line no-unused-vars
@@ -30,6 +31,9 @@ class ConvertBase {
 
         if (!await fs.pathExists(this.pdfToHtmlPath))
             throw new Error('Внешний конвертер pdftohtml не найден');
+
+        if (!await fs.pathExists(this.ddjvuPath))
+            throw new Error('Внешний конвертер ddjvu не найден');
     }
 
     async execConverter(path, args, onData, abort) {
@@ -44,12 +48,18 @@ class ConvertBase {
 
         try {
             const result = await utils.spawnProcess(path, {
-                killAfter: 600,
+                killAfter: 3600,//1 час
                 args, 
                 onData: (data) => {
                     q.resetTimeout();
                     onData(data);
                 },
+                //будем периодически проверять работу конвертера и если очереди нет, то разрешаем работу пинком onData
+                onUsage: (stats) => {
+                    if (queue.freed > 1 && stats.cpu >= 10)
+                        onData('.');
+                },
+                onUsageInterval: 10,
                 abort
             });
             if (result.code != 0) {

server/core/Reader/BookConverter/ConvertDjvu.js

@@ -0,0 +1,39 @@
+const fs = require('fs-extra');
+const path = require('path');
+
+const ConvertPdf = require('./ConvertPdf');
+
+class ConvertRtf extends ConvertPdf {
+    check(data, opts) {
+        const {inputFiles} = opts;
+
+        return this.config.useExternalBookConverter && 
+            inputFiles.sourceFileType && inputFiles.sourceFileType.ext == 'djvu';
+    }
+
+    async run(data, opts) {
+        if (!this.check(data, opts))
+            return false;
+        await this.checkExternalConverterPresent();
+
+        const {inputFiles, callback, abort} = opts;
+
+        const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`;
+        const pdfFile = `${outFile}.pdf`;
+
+        let perc = 0;
+        await this.execConverter(this.ddjvuPath, ['-format=pdf', '-quality=85', '-verbose', inputFiles.sourceFile, pdfFile], () => {
+            perc = (perc < 100 ? perc + 1 : 40);
+            callback(perc);
+        }, abort);
+
+        const pdfFileSize = (await fs.stat(pdfFile)).size;
+        if (pdfFileSize > 2*this.config.maxUploadFileSize) {
+            throw new Error(`Файл для конвертирования слишком большой|FORLOG| ${pdfFileSize} > ${2*this.config.maxUploadFileSize}`);
+        }
+
+        return await super.run(null, Object.assign({}, opts, {pdfFile, skipCheck: true}));
+    }
+}
+
+module.exports = ConvertRtf;

server/core/Reader/BookConverter/ConvertPdf.js

@@ -14,17 +14,20 @@ class ConvertPdf extends ConvertHtml {
     }
 
     async run(notUsed, opts) {
-        if (!this.check(notUsed, opts))
-            return false;
+        if (!opts.skipCheck) {
+            if (!this.check(notUsed, opts))
+                return false;
+        }
         await this.checkExternalConverterPresent();
 
-        const {inputFiles, callback, abort} = opts;
+        const {inputFiles, callback, abort, uploadFileName} = opts;
 
+        const inpFile = (opts.pdfFile ? opts.pdfFile : inputFiles.sourceFile);
         const outFile = `${inputFiles.filesDir}/${utils.randomHexString(10)}.xml`;
 
         //конвертируем в xml
         let perc = 0;
-        await this.execConverter(this.pdfToHtmlPath, ['-nodrm', '-c', '-s', '-xml', inputFiles.sourceFile, outFile], () => {
+        await this.execConverter(this.pdfToHtmlPath, ['-nodrm', '-c', '-s', '-xml', inpFile, outFile], () => {
             perc = (perc < 80 ? perc + 10 : 40);
             callback(perc);
         }, abort);
@@ -186,6 +189,8 @@ class ConvertPdf extends ConvertHtml {
         indents[0] = 0;
 
         //формируем текст
+        if (!title && uploadFileName)
+            title = uploadFileName;
         let text = `<title>${title}</title>`;
         let concat = '';
         let sp = '';

server/core/Reader/BookConverter/index.js

@@ -4,6 +4,7 @@ const FileDetector = require('../../FileDetector');
 //порядок важен
 const convertClassFactory = [
     require('./ConvertEpub'),
+    require('./ConvertDjvu'),
     require('./ConvertPdf'),
     require('./ConvertRtf'),
     require('./ConvertDocX'),

server/core/Reader/ReaderWorker.js

@@ -12,7 +12,7 @@ const utils = require('../utils');
 const log = new (require('../AppLogger'))().log;//singleton
 
 const cleanDirPeriod = 60*60*1000;//1 раз в час
-const queue = new LimitedQueue(5, 100, 5*60*1000);//5 минут ожидание подвижек
+const queue = new LimitedQueue(5, 100, 4*60*1000);//4 минуты ожидание подвижек
 
 let instance = null;
 
@@ -30,7 +30,7 @@ class ReaderWorker {
 
             this.workerState = new WorkerState();
             this.down = new FileDownloader(config.maxUploadFileSize);
-            this.decomp = new FileDecompressor(2*config.maxUploadFileSize);
+            this.decomp = new FileDecompressor(3*config.maxUploadFileSize);
             this.bookConverter = new BookConverter(this.config);
 
             this.remoteWebDavStorage = false;
@@ -81,7 +81,7 @@ class ReaderWorker {
             const decompDirname = utils.randomHexString(30);
 
             //download or use uploaded
-            if (url.indexOf('file://') != 0) {//download
+            if (url.indexOf('disk://') != 0) {//download
                 const downdata = await this.down.load(url, (progress) => {
                     wState.set({progress});
                 }, q.abort);
@@ -130,6 +130,7 @@ class ReaderWorker {
             convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`;
             await this.bookConverter.convertToFb2(decompFiles, convertFilename, opts, progress => {
                 wState.set({progress});
+                q.resetTimeout();
             }, q.abort);
 
             //сжимаем файл в tmp, если там уже нет с тем же именем-sha256
@@ -214,7 +215,7 @@ class ReaderWorker {
             await fs.remove(file.path);
         }
 
-        return `file://${hash}`;
+        return `disk://${hash}`;
     }
 
     async restoreRemoteFile(filename) {

server/core/utils.js

@@ -1,7 +1,9 @@
 const { spawn } = require('child_process');
 const fs = require('fs-extra');
+const path = require('path');
 const crypto = require('crypto');
 const baseX = require('base-x');
+const pidusage = require('pidusage');
 
 const BASE36 = '0123456789abcdefghijklmnopqrstuvwxyz';
 const bs36 = baseX(BASE36);
@@ -45,10 +47,11 @@ async function touchFile(filename) {
 }
 
 function spawnProcess(cmd, opts) {
-    let {args, killAfter, onData, abort} = opts;
+    let {args, killAfter, onData, onUsage, onUsageInterval, abort} = opts;
     killAfter = (killAfter ? killAfter : 120);//seconds
     onData = (onData ? onData : () => {});
     args = (args ? args : []);
+    onUsageInterval = (onUsageInterval ? onUsageInterval : 30);//seconds
 
     return new Promise((resolve, reject) => { (async() => {
         let resolved = false;
@@ -75,9 +78,19 @@ function spawnProcess(cmd, opts) {
             reject({status: 'error', error, stdout, stderr});
         });
 
+        //ждем процесс, контролируем его работу раз в секунду
+        let onUsageCounter = onUsageInterval;
         while (!resolved) {
             await sleep(1000);
-            killAfter -= 1;
+
+            onUsageCounter--;
+            if (onUsage && onUsageCounter <= 0) {
+                const stats = await pidusage(proc.pid);
+                onUsage(stats);
+                onUsageCounter = onUsageInterval;
+            }
+
+            killAfter--;
             if (killAfter <= 0 || (abort && abort())) {
                 process.kill(proc.pid);
                 if (killAfter <= 0) {
@@ -91,6 +104,22 @@ function spawnProcess(cmd, opts) {
     })().catch(reject); });
 }
 
+async function findFiles(callback, dir) {
+    if (!(callback && dir))
+        return;
+    let result = true;
+    const files = await fs.readdir(dir, { withFileTypes: true });
+
+    for (const file of files) {
+        const found = path.resolve(dir, file.name);
+        if (file.isDirectory())
+            result = await findFiles(callback, found);
+        else
+            await callback(found);
+    }
+    return result;
+}
+
 module.exports = {
     toBase36,
     fromBase36,
@@ -99,5 +128,6 @@ module.exports = {
     sleep,
     randomHexString,
     touchFile,
-    spawnProcess
+    spawnProcess,
+    findFiles
 };