Browse Source

Merge branch 'release/0.7.3'

Book Pauk 5 years ago
parent
commit
580b030ee4

+ 2 - 2
client/api/reader.js

@@ -11,11 +11,11 @@ const workerApi = axios.create({
 });
 });
 
 
 class Reader {
 class Reader {
-    async loadBook(url, callback) {
+    async loadBook(opts, callback) {
         const refreshPause = 300;
         const refreshPause = 300;
         if (!callback) callback = () => {};
         if (!callback) callback = () => {};
 
 
-        let response = await api.post('/load-book', {type: 'url', url});
+        let response = await api.post('/load-book', opts);
 
 
         const workerId = response.data.workerId;
         const workerId = response.data.workerId;
         if (!workerId)
         if (!workerId)

+ 2 - 1
client/components/Reader/Reader.vue

@@ -356,6 +356,7 @@ class Reader extends Vue {
         this.showWhatsNewDialog = settings.showWhatsNewDialog;
         this.showWhatsNewDialog = settings.showWhatsNewDialog;
         this.showMigrationDialog = settings.showMigrationDialog;
         this.showMigrationDialog = settings.showMigrationDialog;
         this.showToolButton = settings.showToolButton;
         this.showToolButton = settings.showToolButton;
+        this.enableSitesFilter = settings.enableSitesFilter;
 
 
         this.updateHeaderMinWidth();
         this.updateHeaderMinWidth();
     }
     }
@@ -1003,7 +1004,7 @@ class Reader extends Vue {
             // не удалось, скачиваем книгу полностью с конвертацией
             // не удалось, скачиваем книгу полностью с конвертацией
             let loadCached = true;
             let loadCached = true;
             if (!book) {
             if (!book) {
-                book = await readerApi.loadBook(url, (state) => {
+                book = await readerApi.loadBook({url, enableSitesFilter: this.enableSitesFilter}, (state) => {
                     progress.setState(state);
                     progress.setState(state);
                 });
                 });
                 loadCached = false;
                 loadCached = false;

+ 150 - 140
client/components/Reader/ServerStorage/ServerStorage.vue

@@ -13,6 +13,11 @@ import readerApi from '../../../api/reader';
 import * as utils from '../../../share/utils';
 import * as utils from '../../../share/utils';
 import * as cryptoUtils from '../../../share/cryptoUtils';
 import * as cryptoUtils from '../../../share/cryptoUtils';
 
 
+import localForage from 'localforage';
+const ssCacheStore = localForage.createInstance({
+    name: 'ssCacheStore'
+});
+
 export default @Component({
 export default @Component({
     watch: {
     watch: {
         serverSyncEnabled: function() {
         serverSyncEnabled: function() {
@@ -58,35 +63,52 @@ class ServerStorage extends Vue {
 
 
     async init() {
     async init() {
         try {
         try {
+            this.cachedRecent = await ssCacheStore.getItem('recent');
+            if (!this.cachedRecent)
+                await this.setCachedRecent({rev: 0, data: {}});
+
+            this.cachedRecentPatch = await ssCacheStore.getItem('recent-patch');
+            if (!this.cachedRecentPatch)
+                await this.setCachedRecentPatch({rev: 0, data: {}});
+
+            this.cachedRecentMod = await ssCacheStore.getItem('recent-mod');
+            if (!this.cachedRecentMod)
+                await this.setCachedRecentMod({rev: 0, data: {}});
+
             if (!this.serverStorageKey) {
             if (!this.serverStorageKey) {
                 //генерируем новый ключ
                 //генерируем новый ключ
                 await this.generateNewServerStorageKey();
                 await this.generateNewServerStorageKey();
             } else {
             } else {
                 await this.serverStorageKeyChanged();
                 await this.serverStorageKeyChanged();
             }
             }
+
             bookManager.addEventListener(this.bookManagerEvent);
             bookManager.addEventListener(this.bookManagerEvent);
         } finally {
         } finally {
             this.inited = true;
             this.inited = true;
         }
         }
     }
     }
 
 
+    async setCachedRecent(value) {
+        await ssCacheStore.setItem('recent', value);
+        this.cachedRecent = value;
+    }
+
+    async setCachedRecentPatch(value) {
+        await ssCacheStore.setItem('recent-patch', value);
+        this.cachedRecentPatch = value;
+    }
+
+    async setCachedRecentMod(value) {
+        await ssCacheStore.setItem('recent-mod', value);
+        this.cachedRecentMod = value;
+    }
+
     async bookManagerEvent(eventName, itemKey) {
     async bookManagerEvent(eventName, itemKey) {
         if (!this.serverSyncEnabled)
         if (!this.serverSyncEnabled)
             return;
             return;
 
 
         if (eventName == 'recent-changed') {            
         if (eventName == 'recent-changed') {            
             if (itemKey) {
             if (itemKey) {
-                if (!this.recentDeltaInited) {
-                    await this.loadRecent();
-                    this.warning('Функции сохранения на сервер пока недоступны');
-                    return;
-                }
-
-                if (!this.recentDelta)
-                    this.recentDelta = {};
-
-                this.recentDelta[itemKey] = _.cloneDeep(bookManager.recent[itemKey]);
-
                 this.debouncedSaveRecent(itemKey);
                 this.debouncedSaveRecent(itemKey);
             }
             }
         }
         }
@@ -119,8 +141,8 @@ class ServerStorage extends Vue {
             await this.loadProfiles(force);
             await this.loadProfiles(force);
             this.checkCurrentProfile();
             this.checkCurrentProfile();
             await this.currentProfileChanged(force);
             await this.currentProfileChanged(force);
-            await this.loadRecent();
-            if (force)
+            const loadSuccess = await this.loadRecent();
+            if (loadSuccess && force)
                 await this.saveRecent();
                 await this.saveRecent();
         }
         }
     }
     }
@@ -340,46 +362,26 @@ class ServerStorage extends Vue {
         }
         }
     }
     }
 
 
-    async initRecentDelta() {
-        let recentDelta = null;
-        try {
-            recentDelta = await this.storageGet({recentDelta: {}});
-        } catch(e) {
-            this.error(`Ошибка соединения с сервером: ${e.message}`);
-            return;
-        }
-
-        if (recentDelta.state == 'success') {
-            recentDelta = recentDelta.items.recentDelta;
-
-            if (recentDelta.rev == 0)
-                recentDelta.data = {};
-
-            this.recentDelta = recentDelta.data;
-            this.recentDeltaInited = true;
-        } else {
-            this.warning(`Неверный ответ сервера: ${recentDelta.state}`);
-        }
-    }
-
     async loadRecent(skipRevCheck = false, doNotifySuccess = true) {
     async loadRecent(skipRevCheck = false, doNotifySuccess = true) {
         if (!this.keyInited || !this.serverSyncEnabled || this.loadingRecent)
         if (!this.keyInited || !this.serverSyncEnabled || this.loadingRecent)
             return;
             return;
-
         this.loadingRecent = true;
         this.loadingRecent = true;
         try {
         try {
-            const oldRecentRev = bookManager.recentRev;
-            const oldRecentDeltaRev = bookManager.recentDeltaRev;
             //проверим ревизию на сервере
             //проверим ревизию на сервере
+            let query = {recent: {}, recentPatch: {}, recentMod: {}};
             let revs = null;
             let revs = null;
             if (!skipRevCheck) {
             if (!skipRevCheck) {
                 try {
                 try {
-                    revs = await this.storageCheck({recent: {}, recentDelta: {}});
-                    if (revs.state == 'success' && revs.items.recent.rev == oldRecentRev &&
-                        revs.items.recentDelta.rev == oldRecentDeltaRev) {
-                        if (!this.recentDeltaInited)
-                            await this.initRecentDelta();
-                        return;
+                    revs = await this.storageCheck(query);
+                    if (revs.state == 'success') {
+                        if (revs.items.recent.rev != this.cachedRecent.rev) {
+                            //no changes
+                        } else if (revs.items.recentPatch.rev != this.cachedRecentPatch.rev) {
+                            query = {recentPatch: {}, recentMod: {}};
+                        } else if (revs.items.recentMod.rev != this.cachedRecentMod.rev) {
+                            query = {recentMod: {}};
+                        } else
+                            return true;
                     }
                     }
                 } catch(e) {
                 } catch(e) {
                     this.error(`Ошибка соединения с сервером: ${e.message}`);
                     this.error(`Ошибка соединения с сервером: ${e.message}`);
@@ -389,46 +391,53 @@ class ServerStorage extends Vue {
 
 
             let recent = null;
             let recent = null;
             try {
             try {
-                recent = await this.storageGet({recent: {}, recentDelta: {}});
+                recent = await this.storageGet(query);
             } catch(e) {
             } catch(e) {
                 this.error(`Ошибка соединения с сервером: ${e.message}`);
                 this.error(`Ошибка соединения с сервером: ${e.message}`);
                 return;
                 return;
             }
             }
 
 
             if (recent.state == 'success') {
             if (recent.state == 'success') {
-                let recentDelta = recent.items.recentDelta;
-                recent = recent.items.recent;
-
-                if (recent.rev == 0)
-                    recent.data = {};
-
-                let newRecent = {};
-                if (recentDelta && recentDelta.data) {
-                    if (recentDelta.data.diff) {
-                        newRecent = recent.data;
-                        const key = recentDelta.data.diff.key;
-                        if (newRecent[key])
-                            newRecent[key] = utils.applyObjDiff(newRecent[key], recentDelta.data.diff);
-                    } else {
-                        newRecent = Object.assign(recent.data, recentDelta.data);
-                    }
-                    this.recentDelta = recentDelta.data;
-                } else {
-                    newRecent = recent.data;
-                    this.recentDelta = {};
+                let newRecent = recent.items.recent;
+                let newRecentPatch = recent.items.recentPatch;
+                let newRecentMod = recent.items.recentMod;
+
+                if (!newRecent) {
+                    newRecent = _.cloneDeep(this.cachedRecent);
+                }
+                if (!newRecentPatch) {
+                    newRecentPatch = _.cloneDeep(this.cachedRecentPatch);
+                }
+                if (!newRecentMod) {
+                    newRecentMod = _.cloneDeep(this.cachedRecentMod);
                 }
                 }
 
 
-                this.recentDeltaInited = true;
+                if (newRecent.rev == 0) newRecent.data = {};
+                if (newRecentPatch.rev == 0) newRecentPatch.data = {};
+                if (newRecentMod.rev == 0) newRecentMod.data = {};
+
+                let result = Object.assign({}, newRecent.data, newRecentPatch.data);
+
+                const md = newRecentMod.data;
+                if (md.key && result[md.key])
+                    result[md.key] = utils.applyObjDiff(result[md.key], md.mod);
+
+                if (newRecent.rev != this.cachedRecent.rev)
+                    await this.setCachedRecent(newRecent);
+                if (newRecentPatch.rev != this.cachedRecentPatch.rev)
+                    await this.setCachedRecentPatch(newRecentPatch);
+                if (newRecentMod.rev != this.cachedRecentMod.rev)
+                    await this.setCachedRecentMod(newRecentMod);
 
 
                 if (!bookManager.loaded) {
                 if (!bookManager.loaded) {
                     this.warning('Ожидание загрузки списка книг перед синхронизацией');
                     this.warning('Ожидание загрузки списка книг перед синхронизацией');
                     while (!bookManager.loaded) await utils.sleep(100);
                     while (!bookManager.loaded) await utils.sleep(100);
                 }
                 }
-                await bookManager.setRecent(newRecent);
-                await bookManager.setRecentRev(recent.rev);
-                await bookManager.setRecentDeltaRev(recentDelta.rev);
+
+                await bookManager.setRecent(result);
             } else {
             } else {
                 this.warning(`Неверный ответ сервера: ${recent.state}`);
                 this.warning(`Неверный ответ сервера: ${recent.state}`);
+                return;
             }
             }
 
 
             if (doNotifySuccess)
             if (doNotifySuccess)
@@ -436,6 +445,7 @@ class ServerStorage extends Vue {
         } finally {
         } finally {
             this.loadingRecent = false;
             this.loadingRecent = false;
         }
         }
+        return true;
     }
     }
 
 
     async saveRecent(itemKey, recurse) {
     async saveRecent(itemKey, recurse) {
@@ -444,91 +454,91 @@ class ServerStorage extends Vue {
 
 
         const bm = bookManager;
         const bm = bookManager;
 
 
-        //вычисление критерия сохранения целиком
-        if (!this.sameKeyCount)
-            this.sameKeyCount = 0;
-        if (this.prevItemKey == itemKey) {
-            this.sameKeyCount++;
-        } else {
-            this.sameKeyCount = 0;
-        }
+        let needSaveRecent = false;
+        let needSaveRecentPatch = false;
+        let needSaveRecentMod = false;
 
 
-        const l = Object.keys(this.recentDelta).length - (1*(!!this.recentDelta.diff));
-        this.makeDeltaDiff = (l == 1 && this.prevItemKey == itemKey ? this.makeDeltaDiff : false);
-        const forceSaveRecent =  l > 20 || (this.sameKeyCount > 5 && (l > 1)) || (l == 1 && this.sameKeyCount > 10 && !this.makeDeltaDiff);
+        let applyMod = null;
 
 
-        this.sameKeyCount = (!forceSaveRecent ? this.sameKeyCount : 0);
-        this.prevItemKey = itemKey;
+        //newRecentMod
+        let newRecentMod = {};
+
+        if (itemKey && this.cachedRecentPatch.data[itemKey]) {            
+            if (this.prevItemKey == itemKey) {//сохраняем только дифф
+                newRecentMod = _.cloneDeep(this.cachedRecentMod);
+                newRecentMod.rev++;
 
 
-        //дифф от дельты для уменьшения размера передаваемых данных в частном случае
-        if (this.makeDeltaDiff) {
-            this.recentDelta.diff = utils.getObjDiff(this.prevSavedItem, bm.recent[itemKey]);
-            this.recentDelta.diff.key = itemKey;
-            delete this.recentDelta[itemKey];
-        } else if (this.recentDelta.diff) {
-            const key = this.recentDelta.diff.key;
-            if (!this.prevSavedItem && bm.recent[key])
-                this.prevSavedItem = _.cloneDeep(bm.recent[key]);
-            if (this.prevSavedItem) {
-                this.recentDelta[key] = utils.applyObjDiff(this.prevSavedItem, this.recentDelta.diff);
+                newRecentMod.data.key = itemKey;
+                newRecentMod.data.mod = utils.getObjDiff(this.cachedRecentPatch.data[itemKey], bm.recent[itemKey]);
+                needSaveRecentMod = true;
+            } else {//ключ не совпадает, надо сохранять патч
+                applyMod = newRecentMod.data;
             }
             }
-            delete this.recentDelta.diff;
         }
         }
+        this.prevItemKey = itemKey;
 
 
-        //сохранение
-        this.savingRecent = true;        
-        try {
-            if (forceSaveRecent) {//сохраняем recent целиком
-                let result = {state: ''};
-
-                try {
-                    result = await this.storageSet({recent: {rev: bm.recentRev + 1, data: bm.recent}, recentDelta: {rev: bm.recentDeltaRev + 1, data: {}}});
-                } catch(e) {
-                    this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`);
-                }
-
-                if (result.state == 'reject') {
+        //newRecentPatch
+        let newRecentPatch = {};
+        if (itemKey && !needSaveRecentMod) {
+            newRecentPatch = _.cloneDeep(this.cachedRecentPatch);
+            newRecentPatch.rev++;
+            newRecentPatch.data[itemKey] = bm.recent[itemKey];
+            if (applyMod && applyMod.key && newRecentPatch.data[applyMod.key])
+                newRecentPatch.data[applyMod.key] = utils.applyObjDiff(newRecentPatch.data[applyMod.key], applyMod.mod);
+            newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}};
+            needSaveRecentPatch = true;
+            needSaveRecentMod = true;
+        }
 
 
-                    await this.loadRecent(true, false);
+        //newRecent
+        let newRecent = {};
+        if (!itemKey || (needSaveRecentPatch && Object.keys(newRecentPatch.data).length > 10)) {
+            newRecent = {rev: this.cachedRecent.rev + 1, data: bm.recent};
+            newRecentPatch = {rev: this.cachedRecentPatch.rev + 1, data: {}};
+            newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}};
+            needSaveRecent = true;
+            needSaveRecentPatch = true;
+            needSaveRecentMod = true;
+        }
 
 
-                    this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`);
-                    if (!recurse) {
-                        this.savingRecent = false;
-                        this.recentDelta[itemKey] = _.cloneDeep(bm.recent[itemKey]);
-                        this.saveRecent(itemKey, true);
-                        return;
-                    }
-                } else if (result.state == 'success') {
-                    this.makeDeltaDiff = true;
-                    this.prevSavedItem = _.cloneDeep(bm.recent[itemKey]);
+        //query
+        let query = {};
+        if (needSaveRecent) {
+            query = {recent: newRecent, recentPatch: newRecentPatch, recentMod: newRecentMod};
+        } else if (needSaveRecentPatch) {
+            query = {recentPatch: newRecentPatch, recentMod: newRecentMod};
+        } else {
+            query = {recentMod: newRecentMod};
+        }
 
 
-                    this.recentDelta = {};
-                    await bm.setRecentRev(bm.recentRev + 1);
-                    await bm.setRecentDeltaRev(bm.recentDeltaRev + 1);
-                }
-            } else {//сохраняем только дифф
-                let result = {state: ''};
+        //сохранение
+        this.savingRecent = true;        
+        try {
+            let result = {state: ''};
 
 
-                try {
-                    result = await this.storageSet({recentDelta: {rev: bm.recentDeltaRev + 1, data: this.recentDelta}});
-                } catch(e) {
-                    this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`);
-                }
+            try {
+                result = await this.storageSet(query);
+            } catch(e) {
+                this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`);
+            }
 
 
-                if (result.state == 'reject') {
+            if (result.state == 'reject') {
 
 
-                    await this.loadRecent(true, false);
+                await this.loadRecent(false, false);
 
 
-                    this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`);
-                    if (!recurse) {
-                        this.savingRecent = false;
-                        this.recentDelta[itemKey] = _.cloneDeep(bm.recent[itemKey]);
-                        this.saveRecent(itemKey, true);
-                        return;
-                    }
-                } else if (result.state == 'success') {
-                    await bm.setRecentDeltaRev(bm.recentDeltaRev + 1);
+                this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`);
+                if (!recurse && itemKey) {
+                    this.savingRecent = false;
+                    this.saveRecent(itemKey, true);
+                    return;
                 }
                 }
+            } else if (result.state == 'success') {
+                if (needSaveRecent && newRecent.rev)
+                    await this.setCachedRecent(newRecent);
+                if (needSaveRecentPatch && newRecentPatch.rev)
+                    await this.setCachedRecentPatch(newRecentPatch);
+                if (needSaveRecentMod && newRecentMod.rev)
+                    await this.setCachedRecentMod(newRecentMod);
             }
             }
         } finally {
         } finally {
             this.savingRecent = false;
             this.savingRecent = false;

+ 17 - 0
client/components/Reader/SettingsPage/SettingsPage.vue

@@ -304,6 +304,19 @@
                         </el-col>
                         </el-col>
                         <el-input-number v-model="addEmptyParagraphs" :min="0" :max="2"></el-input-number>
                         <el-input-number v-model="addEmptyParagraphs" :min="0" :max="2"></el-input-number>
                     </el-form-item>
                     </el-form-item>
+                    <el-form-item label="">
+                        <el-tooltip :open-delay="500" effect="light" placement="top">
+                            <template slot="content">
+                                Html-фильтр вырезает лишние элементы со<br>
+                                страницы для определенных сайтов, таких как:<br>
+                                samlib.ru<br>
+                                www.fanfiction.net<br>
+                                archiveofourown.org<br>
+                                и других
+                            </template>
+                            <el-checkbox v-model="enableSitesFilter" @change="needTextReload">Включить html-фильтр для сайтов</el-checkbox>
+                        </el-tooltip>
+                    </el-form-item>
                     
                     
                     <el-form-item label="Изображения">
                     <el-form-item label="Изображения">
                         <el-col :span="11">
                         <el-col :span="11">
@@ -688,6 +701,10 @@ class SettingsPage extends Vue {
         this.$notify.warning({message: 'Необходимо обновить страницу (F5), чтобы изменения возымели эффект'});
         this.$notify.warning({message: 'Необходимо обновить страницу (F5), чтобы изменения возымели эффект'});
     }
     }
 
 
+    needTextReload() {
+        this.$notify.warning({message: 'Необходимо обновить книгу в обход кэша, чтобы изменения возымели эффект'});
+    }
+
     close() {
     close() {
         this.$emit('settings-toggle');
         this.$emit('settings-toggle');
     }
     }

+ 1 - 14
client/components/Reader/share/bookManager.js

@@ -36,9 +36,6 @@ class BookManager {
             }
             }
         }
         }
 
 
-        this.recentRev = await bmRecentStore.getItem('recent-rev') || 0;
-        this.recentDeltaRev = await bmRecentStore.getItem('recent-delta-rev') || 0;
-
         this.recentChanged = true;
         this.recentChanged = true;
 
 
         this.loadStored();//no await
         this.loadStored();//no await
@@ -440,7 +437,7 @@ class BookManager {
 
 
         Object.assign(mergedRecent, value);
         Object.assign(mergedRecent, value);
 
 
-        //подстраховка
+        //подстраховка от hotReload
         for (let i of Object.keys(mergedRecent)) {
         for (let i of Object.keys(mergedRecent)) {
             if (!mergedRecent[i].key || mergedRecent[i].key !== i)
             if (!mergedRecent[i].key || mergedRecent[i].key !== i)
                 delete mergedRecent[i];
                 delete mergedRecent[i];
@@ -466,16 +463,6 @@ class BookManager {
         this.emit('recent-changed');
         this.emit('recent-changed');
     }
     }
 
 
-    async setRecentRev(value) {
-        await bmRecentStore.setItem('recent-rev', value);
-        this.recentRev = value;
-    }
-
-    async setRecentDeltaRev(value) {
-        await bmRecentStore.setItem('recent-delta-rev', value);
-        this.recentDeltaRev = value;
-    }
-
     addEventListener(listener) {
     addEventListener(listener) {
         if (this.eventListeners.indexOf(listener) < 0)
         if (this.eventListeners.indexOf(listener) < 0)
             this.eventListeners.push(listener);        
             this.eventListeners.push(listener);        

+ 4 - 1
client/components/Reader/versionHistory.js

@@ -1,10 +1,13 @@
 export const versionHistory = [
 export const versionHistory = [
 {
 {
     showUntil: '2019-10-17',
     showUntil: '2019-10-17',
-    header: '0.7.2 (2019-10-18)',
+    header: '0.7.3 (2019-10-18)',
     content:
     content:
 `
 `
 <ul>
 <ul>
+    <li>внутренние переделки механизма синхронизации с сервером</li>
+    <li>добавлен html-фильтр для сайтов www.fanfiction.net, archiveofourown.org</li>
+    <li>добавлен параметр "Включить html-фильтр для сайтов" в раздел "Вид"->"Текст" в настройках</li>
     <li>исправления багов</li>
     <li>исправления багов</li>
 </ul>
 </ul>
 `
 `

+ 1 - 0
client/store/modules/reader.js

@@ -183,6 +183,7 @@ const settingDefaults = {
     showServerStorageMessages: true,
     showServerStorageMessages: true,
     showWhatsNewDialog: true,
     showWhatsNewDialog: true,
     showMigrationDialog: true,
     showMigrationDialog: true,
+    enableSitesFilter: true,
 
 
     fontShifts: {},
     fontShifts: {},
     showToolButton: {},
     showToolButton: {},

+ 1 - 1
package.json

@@ -1,6 +1,6 @@
 {
 {
   "name": "Liberama",
   "name": "Liberama",
-  "version": "0.7.2",
+  "version": "0.7.3",
   "engines": {
   "engines": {
     "node": ">=10.0.0"
     "node": ">=10.0.0"
   },
   },

+ 4 - 1
server/controllers/ReaderController.js

@@ -15,7 +15,10 @@ class ReaderController extends BaseController {
         try {
         try {
             if (!request.url) 
             if (!request.url) 
                 throw new Error(`key 'url' is empty`);
                 throw new Error(`key 'url' is empty`);
-            const workerId = this.readerWorker.loadBookUrl(request.url);
+            const workerId = this.readerWorker.loadBookUrl({
+                url: request.url, 
+                enableSitesFilter: (request.hasOwnProperty('enableSitesFilter') ? request.enableSitesFilter : true)
+            });
             const state = workerState.getState(workerId);
             const state = workerState.getState(workerId);
             return (state ? state : {});
             return (state ? state : {});
         } catch (e) {
         } catch (e) {

+ 3 - 0
server/core/BookConverter/ConvertSamlib.js

@@ -20,6 +20,9 @@ class ConvertSamlib extends ConvertBase {
     }
     }
 
 
     async run(data, opts) {
     async run(data, opts) {
+        if (!opts.enableSitesFilter)
+            return false;
+
         const checkResult = this.check(data, opts);
         const checkResult = this.check(data, opts);
         if (!checkResult)
         if (!checkResult)
             return false;
             return false;

+ 71 - 0
server/core/BookConverter/ConvertSites.js

@@ -0,0 +1,71 @@
+const URL = require('url').URL;
+
+const ConvertHtml = require('./ConvertHtml');
+
+const sitesFilter = {
+    'www.fanfiction.net': {
+        converter: 'cutter',
+        begin: `<div class='storytext xcontrast_txt nocopy' id='storytext'>`,
+        end: `<div style='height:5px'></div><div style='clear:both;text-align:right;'>`,
+    },
+    'archiveofourown.org': {
+        converter: 'cutter',
+        begin: `<!-- BEGIN section where work skin applies -->`,
+        end: `<!-- END work skin -->`,
+    }
+};
+
+class ConvertSites extends ConvertHtml {
+    check(data, opts) {        
+        const {url, dataType} = opts;        
+
+        const parsedUrl = new URL(url);
+        if (dataType && dataType.ext == 'html') {
+            if (sitesFilter[parsedUrl.hostname])
+                return {hostname: parsedUrl.hostname};
+        }
+
+        return false;
+    }
+
+    async run(data, opts) {
+        if (!opts.enableSitesFilter)
+            return false;
+        
+        const checkResult = this.check(data, opts);
+        if (!checkResult)
+            return false;
+
+        const {hostname} = checkResult;
+
+        let text = this.decode(data).toString();
+        
+        text = this[sitesFilter[hostname].converter](text, sitesFilter[hostname]);
+        
+        if (text === false)
+            return false;
+
+        return await super.run(Buffer.from(text), {skipCheck: true, cutTitle: true});
+    }
+
+    getTitle(text) {
+        let title = '';
+        const m = text.match(/<title>([\s\S]*?)<\/title>/);
+        if (m)
+            title = m[1];
+
+        return `<title>${title.trim()}</title>`;
+    }
+
+    cutter(text, opts) {
+        const title = this.getTitle(text);
+        const l = text.indexOf(opts.begin);
+        const r = text.indexOf(opts.end);
+        if (l < 0 || r < 0 || r <= l)
+            return false;
+        
+        return text.substring(l, r) + title;
+    }
+}
+
+module.exports = ConvertSites;

+ 6 - 4
server/core/BookConverter/index.js

@@ -11,6 +11,7 @@ const convertClassFactory = [
     require('./ConvertMobi'),
     require('./ConvertMobi'),
     require('./ConvertFb2'),
     require('./ConvertFb2'),
     require('./ConvertSamlib'),
     require('./ConvertSamlib'),
+    require('./ConvertSites'),
     require('./ConvertHtml'),
     require('./ConvertHtml'),
 ];
 ];
 
 
@@ -24,13 +25,14 @@ class BookConverter {
         }
         }
     }
     }
 
 
-    async convertToFb2(inputFiles, outputFile, url, callback) {
+    async convertToFb2(inputFiles, outputFile, opts, callback) {
         const selectedFileType = await this.detector.detectFile(inputFiles.selectedFile);
         const selectedFileType = await this.detector.detectFile(inputFiles.selectedFile);
         const data = await fs.readFile(inputFiles.selectedFile);
         const data = await fs.readFile(inputFiles.selectedFile);
 
 
+        const convertOpts = Object.assign({}, opts, {inputFiles, callback, dataType: selectedFileType});
         let result = false;
         let result = false;
         for (const convert of this.convertFactory) {
         for (const convert of this.convertFactory) {
-            result = await convert.run(data, {inputFiles, url, callback, dataType: selectedFileType});
+            result = await convert.run(data, convertOpts);
             if (result) {
             if (result) {
                 await fs.writeFile(outputFile, result);
                 await fs.writeFile(outputFile, result);
                 break;
                 break;
@@ -38,14 +40,14 @@ class BookConverter {
         }
         }
 
 
         if (!result && inputFiles.nesting) {
         if (!result && inputFiles.nesting) {
-            result = await this.convertToFb2(inputFiles.nesting, outputFile, url, callback);
+            result = await this.convertToFb2(inputFiles.nesting, outputFile, opts, callback);
         }
         }
 
 
         if (!result) {
         if (!result) {
             if (selectedFileType)
             if (selectedFileType)
                 throw new Error(`Этот формат файла не поддерживается: ${selectedFileType.mime}`);
                 throw new Error(`Этот формат файла не поддерживается: ${selectedFileType.mime}`);
             else {
             else {
-                throw new Error(`Не удалось определить формат файла: ${url}`);
+                throw new Error(`Не удалось определить формат файла: ${opts.url}`);
             }
             }
         }
         }
 
 

+ 5 - 4
server/core/ReaderWorker.js

@@ -32,7 +32,8 @@ class ReaderWorker {
         }
         }
     }
     }
 
 
-    async loadBook(url, wState) {
+    async loadBook(opts, wState) {
+        const url = opts.url;
         let errMes = '';
         let errMes = '';
         let decompDir = '';
         let decompDir = '';
         let downloadedFilename = '';
         let downloadedFilename = '';
@@ -77,7 +78,7 @@ class ReaderWorker {
             //конвертирование в fb2
             //конвертирование в fb2
             wState.set({state: 'convert', step: 3, progress: 0});
             wState.set({state: 'convert', step: 3, progress: 0});
             convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`;
             convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`;
-            await this.bookConverter.convertToFb2(decompFiles, convertFilename, url, progress => {
+            await this.bookConverter.convertToFb2(decompFiles, convertFilename, opts, progress => {
                 wState.set({progress});
                 wState.set({progress});
             });
             });
 
 
@@ -105,12 +106,12 @@ class ReaderWorker {
         }
         }
     }
     }
 
 
-    loadBookUrl(url) {
+    loadBookUrl(opts) {
         const workerId = workerState.generateWorkerId();
         const workerId = workerState.generateWorkerId();
         const wState = workerState.getControl(workerId);
         const wState = workerState.getControl(workerId);
         wState.set({state: 'start'});
         wState.set({state: 'start'});
 
 
-        this.loadBook(url, wState);
+        this.loadBook(opts, wState);
 
 
         return workerId;
         return workerId;
     }
     }

+ 1 - 1
server/dev.js

@@ -20,7 +20,7 @@ function webpackDevMiddleware(app) {
 function logQueries(app) {
 function logQueries(app) {
     app.use(function(req, res, next) {
     app.use(function(req, res, next) {
         const start = Date.now();
         const start = Date.now();
-        log(`${req.method} ${req.originalUrl} ${JSON.stringify(req.body).substr(0, 2000)}`);
+        log(`${req.method} ${req.originalUrl} ${JSON.stringify(req.body).substr(0, 4000)}`);
         //log(`${JSON.stringify(req.headers, null, 2)}`)
         //log(`${JSON.stringify(req.headers, null, 2)}`)
         res.once('finish', () => {
         res.once('finish', () => {
             log(`${Date.now() - start}ms`);
             log(`${Date.now() - start}ms`);