Работа над новым поиском

server/config/base.js

@@ -26,6 +26,7 @@ module.exports = {
     cacheCleanInterval: 60,//minutes
     inpxCheckInterval: 60,//minutes
     lowMemoryMode: false,
+    fullOptimization: false,
 
     webConfigParams: ['name', 'version', 'branch', 'bookReadLink', 'dbVersion', 'extendedSearch'],
 

server/config/index.js

@@ -15,6 +15,7 @@ const propsToSave = [
     'cacheCleanInterval',
     'inpxCheckInterval',
     'lowMemoryMode',
+    'fullOptimization',
     'allowRemoteLib',
     'remoteLib',
     'server',

server/core/DbCreator.js

@@ -138,7 +138,7 @@ class DbCreator {
                 callback({progress: (readState.current || 0)/totalFiles});
         };
 
-        const parseField = (fieldValue, fieldMap, fieldArr, bookId, fillBookIds = true) => {
+        const parseField = (fieldValue, fieldMap, fieldArr, bookId, rec, fillBookIds = true) => {
             let value = fieldValue;
 
             if (typeof(fieldValue) == 'string') {
@@ -154,12 +154,24 @@ class DbCreator {
                 fieldRec = fieldArr[fieldId];
             } else {
                 fieldRec = {id: fieldArr.length, value, bookIds: new Set()};                
+                if (rec !== undefined) {
+                    fieldRec.name = fieldValue;
+                    fieldRec.bookCount = 0;
+                    fieldRec.bookDelCount = 0;
+                }
                 fieldArr.push(fieldRec);
                 fieldMap.set(value, fieldRec.id);
             }
 
             if (fieldValue !== emptyFieldValue || fillBookIds)
                 fieldRec.bookIds.add(bookId);
+
+            if (rec !== undefined) {
+                if (!rec.del)
+                    fieldRec.bookCount++;
+                else
+                    fieldRec.bookDelCount++;
+            }
         };        
 
         const parseBookRec = (rec) => {
@@ -173,14 +185,14 @@ class DbCreator {
                 if (!authorMap.has(a.toLowerCase()) && (author.length == 1 || i < author.length - 1)) //без соавторов
                     authorCount++;
                 
-                parseField(a, authorMap, authorArr, rec.id);                
+                parseField(a, authorMap, authorArr, rec.id, rec);                
             }
 
             //серии
-            parseField(rec.series, seriesMap, seriesArr, rec.id, false);
+            parseField(rec.series, seriesMap, seriesArr, rec.id, rec, false);
 
             //названия
-            parseField(rec.title, titleMap, titleArr, rec.id);
+            parseField(rec.title, titleMap, titleArr, rec.id, rec);
 
             //жанры
             let genre = rec.genre || emptyFieldValue;
@@ -393,10 +405,12 @@ class DbCreator {
         await db.create({table: 'file_hash'});
 
         //-- завершающие шаги --------------------------------
-        await db.open({
-            table: 'book',
-            cacheSize: (config.lowMemoryMode ? 5 : 500),
-        });
+        if (config.fullOptimization) {
+            await db.open({
+                table: 'book',
+                cacheSize: (config.lowMemoryMode ? 5 : 500),
+            });
+        }
 
         callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0});
         await this.optimizeTable('author', db, (p) => {
@@ -419,7 +433,8 @@ class DbCreator {
         await this.countStats(db, callback, stats);
 
         //чистка памяти, ибо жрет как не в себя
-        await db.drop({table: 'book'});//больше не понадобится
+        if (config.fullOptimization)
+            await db.close({table: 'book'});
         await db.freeMemory();
         utils.freeMemory();
 
@@ -440,17 +455,13 @@ class DbCreator {
     }
 
     async optimizeTable(from, db, callback) {
+        const config = this.config;
+
         const to = `${from}_book`;
         const toId = `${from}_id`;
-        const restoreProp = from;
 
-        //оптимизация таблицы from, превращаем массив bookId в books, кладем все в таблицу to
         await db.open({table: from});
-
-        await db.create({
-            table: to,
-            flag: {name: 'toDel', check: 'r => r.toDel'},
-        });
+        await db.create({table: to});
 
         const bookId2RecId = new Map();
 
@@ -469,46 +480,35 @@ class DbCreator {
                 }
             }
 
-            ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb
+            if (config.fullOptimization) {
+                ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb
 
-            const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
+                const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
 
-            const bookArr = new Map();
-            for (const row of rows)
-                bookArr.set(row.id, row);
+                const bookArr = new Map();
+                for (const row of rows)
+                    bookArr.set(row.id, row);
 
-            for (const rec of chunk) {
-                rec.books = [];
-                rec.bookCount = 0;
-                rec.bookDelCount = 0;
+                for (const rec of chunk) {
+                    rec.books = [];
 
-                for (const id of rec.bookIds) {
-                    const book = bookArr.get(id);
-                    if (rec) {//на всякий случай
-                        rec.books.push(book);
-                        if (!book.del)
-                            rec.bookCount++;
-                        else
-                            rec.bookDelCount++;
+                    for (const id of rec.bookIds) {
+                        const book = bookArr.get(id);
+                        if (book) {//на всякий случай
+                            rec.books.push(book);
+                        }
                     }
-                }
 
-                if (rec.books.length) {
-                    rec[restoreProp] = rec.value;//rec.books[0][restoreProp];
-                    if (!rec[restoreProp])
-                        rec[restoreProp] = emptyFieldValue;
-                } else {
-                    rec.toDel = 1;
+                    delete rec.name;
+                    delete rec.value;
+                    delete rec.bookIds;
                 }
 
-                delete rec.value;
-                delete rec.bookIds;
+                await db.insert({
+                    table: to,
+                    rows: chunk,
+                });
             }
-
-            await db.insert({
-                table: to,
-                rows: chunk,
-            });
         };
 
         const rows = await db.select({table: from, count: true});
@@ -558,7 +558,6 @@ class DbCreator {
             }
         }
 
-        await db.delete({table: to, where: `@@flag('toDel')`});
         await db.close({table: to});
         await db.close({table: from});
 

server/core/DbSearcher.js

@@ -500,76 +500,45 @@ class DbSearcher {
         return tableIds;
     }
 
-    async getCached(key) {
-        if (!this.config.queryCacheEnabled)
-            return null;
-
-        let result = null;
-
+    async restoreBooks(from, ids) {
         const db = this.db;
-        const memCache = this.memCache;
+        const bookTable = `${from}_book`;
 
-        if (memCache.has(key)) {//есть в недавних
-            result = memCache.get(key);
+        const rows = await db.select({
+            table: bookTable,
+            where: `@@id(${db.esc(ids)})`
+        });
 
-            //изменим порядок ключей, для последующей правильной чистки старых
-            memCache.delete(key);
-            memCache.set(key, result);
-        } else {//смотрим в таблице
-            const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`});
+        if (rows.length == ids.length)
+            return rows;
 
-            if (rows.length) {//нашли в кеше
-                await db.insert({
-                    table: 'query_time',
-                    replace: true,
-                    rows: [{id: key, time: Date.now()}],
-                });
+        const idsSet = new Set(rows.map(r => r.id));
 
-                result = rows[0].value;
-                memCache.set(key, result);
-
-                if (memCache.size > maxMemCacheSize) {
-                    //удаляем самый старый ключ-значение
-                    for (const k of memCache.keys()) {
-                        memCache.delete(k);
-                        break;
-                    }
-                }
-            }
-        }
+        for (const id of ids) {
+            if (!idsSet.has(id)) {
+                const bookIds = await db.select({
+                    table: from,
+                    where: `@@id(${db.esc(id)})`
+                });
 
-        return result;
-    }
+                if (!bookIds.length)
+                    continue;
 
-    async putCached(key, value) {
-        if (!this.config.queryCacheEnabled)
-            return;
+                let books = await db.select({
+                    table: 'book',
+                    where: `@@id(${db.esc(bookIds[0].bookIds)})`
+                });
 
-        const db = this.db;
+                if (!books.length)
+                    continue;
 
-        const memCache = this.memCache;
-        memCache.set(key, value);
+                rows.push({id, name: bookIds[0].name, books});
 
-        if (memCache.size > maxMemCacheSize) {
-            //удаляем самый старый ключ-значение
-            for (const k of memCache.keys()) {
-                memCache.delete(k);
-                break;
+                await db.insert({table: bookTable, ignore: true, rows});
             }
         }
 
-        //кладем в таблицу
-        await db.insert({
-            table: 'query_cache',
-            replace: true,
-            rows: [{id: key, value}],
-        });
-
-        await db.insert({
-            table: 'query_time',
-            replace: true,
-            rows: [{id: key, time: Date.now()}],
-        });
+        return rows;
     }
 
     async search(from, query) {
@@ -591,10 +560,10 @@ class DbSearcher {
             limit = (limit > maxLimit ? maxLimit : limit);
             const offset = (query.offset ? query.offset : 0);
 
-            //выборка найденных авторов
+            //выборка найденных значений
             const found = await db.select({
-                table: `${from}_book`,
-                map: `(r) => ({id: r.id, ${from}: r.${from}, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
+                table: from,
+                map: `(r) => ({id: r.id, ${from}: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
                 where: `@@id(${db.esc(ids.slice(offset, offset + limit))})`
             });
 
@@ -614,19 +583,14 @@ class DbSearcher {
         this.searchFlag++;
 
         try {
-            const db = this.db;
-
             //выборка книг автора по authorId
-            const rows = await db.select({
-                table: 'author_book',
-                where: `@@id(${db.esc(authorId)})`
-            });
+            const rows = await this.restoreBooks('author', [authorId])
 
             let author = '';
             let books = '';
 
             if (rows.length) {
-                author = rows[0].author;
+                author = rows[0].name;
                 books = rows[0].books;
             }
 
@@ -660,10 +624,7 @@ class DbSearcher {
             let books;
             if (rows.length && rows[0].rawResult.length) {
                 //выборка книг серии
-                rows = await db.select({
-                    table: 'series_book',
-                    where: `@@id(${rows[0].rawResult[0]})`
-                });
+                const rows = await this.restoreBooks('series', [rows[0].rawResult[0]])
 
                 if (rows.length)
                     books = rows[0].books;
@@ -675,6 +636,78 @@ class DbSearcher {
         }
     }
 
+    async getCached(key) {
+        if (!this.config.queryCacheEnabled)
+            return null;
+
+        let result = null;
+
+        const db = this.db;
+        const memCache = this.memCache;
+
+        if (memCache.has(key)) {//есть в недавних
+            result = memCache.get(key);
+
+            //изменим порядок ключей, для последующей правильной чистки старых
+            memCache.delete(key);
+            memCache.set(key, result);
+        } else {//смотрим в таблице
+            const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`});
+
+            if (rows.length) {//нашли в кеше
+                await db.insert({
+                    table: 'query_time',
+                    replace: true,
+                    rows: [{id: key, time: Date.now()}],
+                });
+
+                result = rows[0].value;
+                memCache.set(key, result);
+
+                if (memCache.size > maxMemCacheSize) {
+                    //удаляем самый старый ключ-значение
+                    for (const k of memCache.keys()) {
+                        memCache.delete(k);
+                        break;
+                    }
+                }
+            }
+        }
+
+        return result;
+    }
+
+    async putCached(key, value) {
+        if (!this.config.queryCacheEnabled)
+            return;
+
+        const db = this.db;
+
+        const memCache = this.memCache;
+        memCache.set(key, value);
+
+        if (memCache.size > maxMemCacheSize) {
+            //удаляем самый старый ключ-значение
+            for (const k of memCache.keys()) {
+                memCache.delete(k);
+                break;
+            }
+        }
+
+        //кладем в таблицу
+        await db.insert({
+            table: 'query_cache',
+            replace: true,
+            rows: [{id: key, value}],
+        });
+
+        await db.insert({
+            table: 'query_time',
+            replace: true,
+            rows: [{id: key, time: Date.now()}],
+        });
+    }
+
     async periodicCleanCache() {
         this.timer = null;
         const cleanInterval = this.config.cacheCleanInterval*60*1000;