DbSearcher.js 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999
  1. const fs = require('fs-extra');
  2. //const _ = require('lodash');
  3. const utils = require('./utils');
  4. const maxLimit = 1000;
  5. const emptyFieldValue = '?';
  6. const maxUtf8Char = String.fromCodePoint(0xFFFFF);
  7. const ruAlphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя';
  8. const enAlphabet = 'abcdefghijklmnopqrstuvwxyz';
  9. const enruArr = (ruAlphabet + enAlphabet).split('');
  10. class DbSearcher {
  11. constructor(config, db) {
  12. this.config = config;
  13. this.queryCacheMemSize = this.config.queryCacheMemSize;
  14. this.queryCacheDiskSize = this.config.queryCacheDiskSize;
  15. this.queryCacheEnabled = this.config.queryCacheEnabled
  16. && (this.queryCacheMemSize > 0 || this.queryCacheDiskSize > 0);
  17. this.db = db;
  18. this.searchFlag = 0;
  19. this.timer = null;
  20. this.closed = false;
  21. this.memCache = new Map();
  22. this.bookIdMap = {};
  23. this.periodicCleanCache();//no await
  24. }
  25. async init() {
  26. await this.fillBookIdMap('author');
  27. await this.fillBookIdMap('series');
  28. await this.fillBookIdMap('title');
  29. await this.fillDbConfig();
  30. }
  31. queryKey(q) {
  32. const result = [];
  33. for (const f of this.recStruct) {
  34. result.push(q[f.field]);
  35. }
  36. return JSON.stringify(result);
  37. }
  38. getWhere(a) {
  39. const db = this.db;
  40. a = a.toLowerCase();
  41. let where;
  42. //особая обработка префиксов
  43. if (a[0] == '=') {
  44. a = a.substring(1);
  45. where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a)})`;
  46. } else if (a[0] == '*') {
  47. a = a.substring(1);
  48. where = `@indexIter('value', (v) => (v !== ${db.esc(emptyFieldValue)} && v.indexOf(${db.esc(a)}) >= 0) )`;
  49. } else if (a[0] == '#') {
  50. a = a.substring(1);
  51. where = `@indexIter('value', (v) => {
  52. const enru = new Set(${db.esc(enruArr)});
  53. return !v || (v !== ${db.esc(emptyFieldValue)} && !enru.has(v[0]) && v.indexOf(${db.esc(a)}) >= 0);
  54. })`;
  55. } else {
  56. where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`;
  57. }
  58. return where;
  59. }
  60. async selectBookIds(query) {
  61. const db = this.db;
  62. const idsArr = [];
  63. const tableBookIds = async(table, where) => {
  64. const rows = await db.select({
  65. table,
  66. rawResult: true,
  67. where: `
  68. const ids = ${where};
  69. const result = new Set();
  70. for (const id of ids) {
  71. const row = @unsafeRow(id);
  72. for (const bookId of row.bookIds)
  73. result.add(bookId);
  74. }
  75. return new Uint32Array(result);
  76. `
  77. });
  78. return rows[0].rawResult;
  79. };
  80. //авторы
  81. if (query.author && query.author !== '*') {
  82. const key = `book-ids-author-${query.author}`;
  83. let ids = await this.getCached(key);
  84. if (ids === null) {
  85. ids = await tableBookIds('author', this.getWhere(query.author));
  86. await this.putCached(key, ids);
  87. }
  88. idsArr.push(ids);
  89. }
  90. //серии
  91. if (query.series && query.series !== '*') {
  92. const key = `book-ids-series-${query.series}`;
  93. let ids = await this.getCached(key);
  94. if (ids === null) {
  95. ids = await tableBookIds('series', this.getWhere(query.series));
  96. await this.putCached(key, ids);
  97. }
  98. idsArr.push(ids);
  99. }
  100. //названия
  101. if (query.title && query.title !== '*') {
  102. const key = `book-ids-title-${query.title}`;
  103. let ids = await this.getCached(key);
  104. if (ids === null) {
  105. ids = await tableBookIds('title', this.getWhere(query.title));
  106. await this.putCached(key, ids);
  107. }
  108. idsArr.push(ids);
  109. }
  110. //жанры
  111. if (query.genre) {
  112. const key = `book-ids-genre-${query.genre}`;
  113. let ids = await this.getCached(key);
  114. if (ids === null) {
  115. const genreRows = await db.select({
  116. table: 'genre',
  117. rawResult: true,
  118. where: `
  119. const genres = ${db.esc(query.genre.split(','))};
  120. const ids = new Set();
  121. for (const g of genres) {
  122. for (const id of @indexLR('value', g, g))
  123. ids.add(id);
  124. }
  125. const result = new Set();
  126. for (const id of ids) {
  127. const row = @unsafeRow(id);
  128. for (const bookId of row.bookIds)
  129. result.add(bookId);
  130. }
  131. return new Uint32Array(result);
  132. `
  133. });
  134. ids = genreRows[0].rawResult;
  135. await this.putCached(key, ids);
  136. }
  137. idsArr.push(ids);
  138. }
  139. //языки
  140. if (query.lang) {
  141. const key = `book-ids-lang-${query.lang}`;
  142. let ids = await this.getCached(key);
  143. if (ids === null) {
  144. const langRows = await db.select({
  145. table: 'lang',
  146. rawResult: true,
  147. where: `
  148. const langs = ${db.esc(query.lang.split(','))};
  149. const ids = new Set();
  150. for (const l of langs) {
  151. for (const id of @indexLR('value', l, l))
  152. ids.add(id);
  153. }
  154. const result = new Set();
  155. for (const id of ids) {
  156. const row = @unsafeRow(id);
  157. for (const bookId of row.bookIds)
  158. result.add(bookId);
  159. }
  160. return new Uint32Array(result);
  161. `
  162. });
  163. ids = langRows[0].rawResult;
  164. await this.putCached(key, ids);
  165. }
  166. idsArr.push(ids);
  167. }
  168. //удаленные
  169. if (query.del) {
  170. const del = parseInt(query.del, 10) || 0;
  171. const key = `book-ids-del-${del}`;
  172. let ids = await this.getCached(key);
  173. if (ids === null) {
  174. ids = await tableBookIds('del', `@indexLR('value', ${db.esc(del)}, ${db.esc(del)})`);
  175. await this.putCached(key, ids);
  176. }
  177. idsArr.push(ids);
  178. }
  179. //дата поступления
  180. if (query.date) {
  181. const key = `book-ids-date-${query.date}`;
  182. let ids = await this.getCached(key);
  183. if (ids === null) {
  184. let [from = '', to = ''] = query.date.split(',');
  185. ids = await tableBookIds('date', `@indexLR('value', ${db.esc(from)} || undefined, ${db.esc(to)} || undefined)`);
  186. await this.putCached(key, ids);
  187. }
  188. idsArr.push(ids);
  189. }
  190. //оценка
  191. if (query.librate) {
  192. const key = `book-ids-librate-${query.librate}`;
  193. let ids = await this.getCached(key);
  194. if (ids === null) {
  195. const dateRows = await db.select({
  196. table: 'librate',
  197. rawResult: true,
  198. where: `
  199. const rates = ${db.esc(query.librate.split(',').map(n => parseInt(n, 10)).filter(n => !isNaN(n)))};
  200. const ids = new Set();
  201. for (const rate of rates) {
  202. for (const id of @indexLR('value', rate, rate))
  203. ids.add(id);
  204. }
  205. const result = new Set();
  206. for (const id of ids) {
  207. const row = @unsafeRow(id);
  208. for (const bookId of row.bookIds)
  209. result.add(bookId);
  210. }
  211. return new Uint32Array(result);
  212. `
  213. });
  214. ids = dateRows[0].rawResult;
  215. await this.putCached(key, ids);
  216. }
  217. idsArr.push(ids);
  218. }
  219. if (idsArr.length > 1) {
  220. //ищем пересечение множеств
  221. let proc = 0;
  222. let nextProc = 0;
  223. let inter = new Set(idsArr[0]);
  224. for (let i = 1; i < idsArr.length; i++) {
  225. const newInter = new Set();
  226. for (const id of idsArr[i]) {
  227. if (inter.has(id))
  228. newInter.add(id);
  229. //прерываемся иногда, чтобы не блокировать Event Loop
  230. proc++;
  231. if (proc >= nextProc) {
  232. nextProc += 10000;
  233. await utils.processLoop();
  234. }
  235. }
  236. inter = newInter;
  237. }
  238. return new Uint32Array(inter);
  239. } else if (idsArr.length == 1) {
  240. return idsArr[0];
  241. } else {
  242. return false;
  243. }
  244. }
  245. async fillDbConfig() {
  246. if (!this.dbConfig) {
  247. const rows = await this.db.select({table: 'config'});
  248. const config = {};
  249. for (const row of rows) {
  250. config[row.id] = row.value;
  251. }
  252. this.dbConfig = config;
  253. this.recStruct = config.inpxInfo.recStruct;
  254. }
  255. }
  256. async fillBookIdMap(from) {
  257. const data = await fs.readFile(`${this.config.dataDir}/db/${from}_id.map`, 'utf-8');
  258. const idMap = JSON.parse(data);
  259. idMap.arr = new Uint32Array(idMap.arr);
  260. idMap.map = new Map(idMap.map);
  261. this.bookIdMap[from] = idMap;
  262. }
  263. async tableIdsFilter(from, query) {
  264. //т.к. авторы у книги идут списком (т.е. одна книга относиться сразу к нескольким авторам),
  265. //то в выборку по bookId могут попасть авторы, которые отсутствуют в критерии query.author,
  266. //поэтому дополнительно фильтруем
  267. let result = null;
  268. if (from == 'author' && query.author && query.author !== '*') {
  269. const key = `filter-ids-author-${query.author}`;
  270. let authorIds = await this.getCached(key);
  271. if (authorIds === null) {
  272. const rows = await this.db.select({
  273. table: 'author',
  274. rawResult: true,
  275. where: `return new Uint32Array(${this.getWhere(query.author)})`
  276. });
  277. authorIds = rows[0].rawResult;
  278. await this.putCached(key, authorIds);
  279. }
  280. result = new Set(authorIds);
  281. }
  282. return result;
  283. }
  284. async selectTableIds(from, query) {
  285. const db = this.db;
  286. const queryKey = this.queryKey(query);
  287. const tableKey = `${from}-table-ids-${queryKey}`;
  288. let tableIds = await this.getCached(tableKey);
  289. if (tableIds === null) {
  290. const bookKey = `book-ids-${queryKey}`;
  291. let bookIds = await this.getCached(bookKey);
  292. if (bookIds === null) {
  293. bookIds = await this.selectBookIds(query);
  294. await this.putCached(bookKey, bookIds);
  295. }
  296. //id книг (bookIds) нашли, теперь надо их смаппировать в id таблицы from (авторов, серий, названий)
  297. if (bookIds) {
  298. //т.к. авторы у книги идут списком, то дополнительно фильтруем
  299. const filter = await this.tableIdsFilter(from, query);
  300. const tableIdsSet = new Set();
  301. const idMap = this.bookIdMap[from];
  302. let proc = 0;
  303. let nextProc = 0;
  304. for (const bookId of bookIds) {
  305. const tableId = idMap.arr[bookId];
  306. if (tableId) {
  307. if (!filter || filter.has(tableId))
  308. tableIdsSet.add(tableId);
  309. proc++;
  310. } else {
  311. const tableIdArr = idMap.map.get(bookId);
  312. if (tableIdArr) {
  313. for (const tableId of tableIdArr) {
  314. if (!filter || filter.has(tableId))
  315. tableIdsSet.add(tableId);
  316. proc++;
  317. }
  318. }
  319. }
  320. //прерываемся иногда, чтобы не блокировать Event Loop
  321. if (proc >= nextProc) {
  322. nextProc += 10000;
  323. await utils.processLoop();
  324. }
  325. }
  326. tableIds = new Uint32Array(tableIdsSet);
  327. } else {//bookIds пустой - критерии не заданы, значит берем все id из from
  328. const rows = await db.select({
  329. table: from,
  330. rawResult: true,
  331. where: `return new Uint32Array(@all())`
  332. });
  333. tableIds = rows[0].rawResult;
  334. }
  335. //сортируем по id
  336. //порядок id соответствует ASC-сортировке по строковому значению из from (имя автора, назание серии, название книги)
  337. tableIds.sort((a, b) => a - b);
  338. await this.putCached(tableKey, tableIds);
  339. }
  340. return tableIds;
  341. }
  342. async restoreBooks(from, ids) {
  343. const db = this.db;
  344. const bookTable = `${from}_book`;
  345. const rows = await db.select({
  346. table: bookTable,
  347. where: `@@id(${db.esc(ids)})`
  348. });
  349. if (rows.length == ids.length)
  350. return rows;
  351. //далее восстановим книги из book в <from>_book
  352. const idsSet = new Set(rows.map(r => r.id));
  353. //недостающие
  354. const tableIds = [];
  355. for (const id of ids) {
  356. if (!idsSet.has(id))
  357. tableIds.push(id);
  358. }
  359. const tableRows = await db.select({
  360. table: from,
  361. where: `@@id(${db.esc(tableIds)})`
  362. });
  363. //список недостающих bookId
  364. const bookIds = [];
  365. for (const row of tableRows) {
  366. for (const bookId of row.bookIds)
  367. bookIds.push(bookId);
  368. }
  369. //выбираем книги
  370. const books = await db.select({
  371. table: 'book',
  372. where: `@@id(${db.esc(bookIds)})`
  373. });
  374. const booksMap = new Map();
  375. for (const book of books)
  376. booksMap.set(book.id, book);
  377. //распределяем
  378. for (const row of tableRows) {
  379. const books = [];
  380. for (const bookId of row.bookIds) {
  381. const book = booksMap.get(bookId);
  382. if (book)
  383. books.push(book);
  384. }
  385. rows.push({id: row.id, name: row.name, books});
  386. }
  387. await db.insert({table: bookTable, ignore: true, rows});
  388. return rows;
  389. }
  390. async search(from, query) {
  391. if (this.closed)
  392. throw new Error('DbSearcher closed');
  393. if (!['author', 'series', 'title'].includes(from))
  394. throw new Error(`Unknown value for param 'from'`);
  395. this.searchFlag++;
  396. try {
  397. const db = this.db;
  398. const ids = await this.selectTableIds(from, query);
  399. const totalFound = ids.length;
  400. let limit = (query.limit ? query.limit : 100);
  401. limit = (limit > maxLimit ? maxLimit : limit);
  402. const offset = (query.offset ? query.offset : 0);
  403. const slice = ids.slice(offset, offset + limit);
  404. //выборка найденных значений
  405. const found = await db.select({
  406. table: from,
  407. map: `(r) => ({id: r.id, ${from}: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
  408. where: `@@id(${db.esc(Array.from(slice))})`
  409. });
  410. //для title восстановим books
  411. if (from == 'title') {
  412. const bookIds = found.map(r => r.id);
  413. const rows = await this.restoreBooks(from, bookIds);
  414. const rowsMap = new Map();
  415. for (const row of rows)
  416. rowsMap.set(row.id, row);
  417. for (const f of found) {
  418. const b = rowsMap.get(f.id);
  419. if (b)
  420. f.books = b.books;
  421. }
  422. }
  423. return {found, totalFound};
  424. } finally {
  425. this.searchFlag--;
  426. }
  427. }
  428. async bookSearchIds(query) {
  429. const queryKey = this.queryKey(query);
  430. const bookKey = `book-search-ids-${queryKey}`;
  431. let bookIds = await this.getCached(bookKey);
  432. if (bookIds === null) {
  433. const db = this.db;
  434. const filterBySearch = (bookField, searchValue) => {
  435. searchValue = searchValue.toLowerCase();
  436. //особая обработка префиксов
  437. if (searchValue[0] == '=') {
  438. searchValue = searchValue.substring(1);
  439. return `(row.${bookField}.toLowerCase().localeCompare(${db.esc(searchValue)}) === 0)`;
  440. } else if (searchValue[0] == '*') {
  441. searchValue = searchValue.substring(1);
  442. return `(row.${bookField} && row.${bookField}.toLowerCase().indexOf(${db.esc(searchValue)}) >= 0)`;
  443. } else if (searchValue[0] == '#') {
  444. searchValue = searchValue.substring(1);
  445. return `(row.${bookField} === '' || (!enru.has(row.${bookField}.toLowerCase()[0]) && row.${bookField}.toLowerCase().indexOf(${db.esc(searchValue)}) >= 0))`;
  446. } else {
  447. return `(row.${bookField}.toLowerCase().localeCompare(${db.esc(searchValue)}) >= 0 ` +
  448. `&& row.${bookField}.toLowerCase().localeCompare(${db.esc(searchValue)} + ${db.esc(maxUtf8Char)}) <= 0)`;
  449. }
  450. };
  451. const checks = ['true'];
  452. for (const f of this.recStruct) {
  453. if (query[f.field]) {
  454. let searchValue = query[f.field];
  455. if (f.type === 'S') {
  456. checks.push(filterBySearch(f.field, searchValue));
  457. } if (f.type === 'N') {
  458. const v = searchValue.split('..');
  459. if (v.length == 1) {
  460. searchValue = parseInt(searchValue, 10);
  461. if (isNaN(searchValue))
  462. throw new Error(`Wrong query param, ${f.field}=${searchValue}`);
  463. checks.push(`row.${f.field} === ${searchValue}`);
  464. } else {
  465. const from = parseInt(v[0] || '0', 10);
  466. const to = parseInt(v[1] || '0', 10);
  467. if (isNaN(from) || isNaN(to))
  468. throw new Error(`Wrong query param, ${f.field}=${searchValue}`);
  469. checks.push(`row.${f.field} >= ${from} && row.${f.field} <= ${to}`);
  470. }
  471. }
  472. }
  473. }
  474. const rows = await db.select({
  475. table: 'book',
  476. rawResult: true,
  477. where: `
  478. const enru = new Set(${db.esc(enruArr)});
  479. const checkBook = (row) => {
  480. return ${checks.join(' && ')};
  481. };
  482. const result = [];
  483. for (const id of @all()) {
  484. const row = @unsafeRow(id);
  485. if (checkBook(row))
  486. result.push(row.id);
  487. }
  488. return new Uint32Array(result);
  489. `
  490. });
  491. bookIds = rows[0].rawResult;
  492. await this.putCached(bookKey, bookIds);
  493. }
  494. return bookIds;
  495. }
  496. //неоптимизированный поиск по всем книгам, по всем полям
  497. async bookSearch(query) {
  498. if (this.closed)
  499. throw new Error('DbSearcher closed');
  500. this.searchFlag++;
  501. try {
  502. const db = this.db;
  503. const ids = await this.bookSearchIds(query);
  504. const totalFound = ids.length;
  505. let limit = (query.limit ? query.limit : 100);
  506. limit = (limit > maxLimit ? maxLimit : limit);
  507. const offset = (query.offset ? query.offset : 0);
  508. const slice = ids.slice(offset, offset + limit);
  509. //выборка найденных значений
  510. const found = await db.select({
  511. table: 'book',
  512. where: `@@id(${db.esc(Array.from(slice))})`
  513. });
  514. return {found, totalFound};
  515. } finally {
  516. this.searchFlag--;
  517. }
  518. }
  519. async opdsQuery(from, query) {
  520. if (this.closed)
  521. throw new Error('DbSearcher closed');
  522. if (!['author', 'series', 'title'].includes(from))
  523. throw new Error(`Unknown value for param 'from'`);
  524. this.searchFlag++;
  525. try {
  526. const db = this.db;
  527. const depth = query.depth || 1;
  528. const queryKey = this.queryKey(query);
  529. const opdsKey = `${from}-opds-d${depth}-${queryKey}`;
  530. let result = await this.getCached(opdsKey);
  531. if (result === null) {
  532. const ids = await this.selectTableIds(from, query);
  533. const totalFound = ids.length;
  534. //группировка по name длиной depth
  535. const found = await db.select({
  536. table: from,
  537. rawResult: true,
  538. where: `
  539. const depth = ${db.esc(depth)};
  540. const group = new Map();
  541. const ids = ${db.esc(Array.from(ids))};
  542. for (const id of ids) {
  543. const row = @unsafeRow(id);
  544. const s = row.value.substring(0, depth);
  545. let g = group.get(s);
  546. if (!g) {
  547. g = {id: row.id, name: row.name, value: s, count: 0, bookCount: 0};
  548. group.set(s, g);
  549. }
  550. g.count++;
  551. g.bookCount += row.bookCount;
  552. }
  553. const result = Array.from(group.values());
  554. result.sort((a, b) => a.value.localeCompare(b.value));
  555. return result;
  556. `
  557. });
  558. result = {found: found[0].rawResult, totalFound};
  559. await this.putCached(opdsKey, result);
  560. }
  561. return result;
  562. } finally {
  563. this.searchFlag--;
  564. }
  565. }
  566. async getAuthorBookList(authorId, author) {
  567. if (this.closed)
  568. throw new Error('DbSearcher closed');
  569. if (!authorId && !author)
  570. return {author: '', books: []};
  571. this.searchFlag++;
  572. try {
  573. const db = this.db;
  574. if (!authorId) {
  575. //восстановим authorId
  576. authorId = 0;
  577. author = author.toLowerCase();
  578. const rows = await db.select({
  579. table: 'author',
  580. rawResult: true,
  581. where: `return Array.from(@dirtyIndexLR('value', ${db.esc(author)}, ${db.esc(author)}))`
  582. });
  583. if (rows.length && rows[0].rawResult.length)
  584. authorId = rows[0].rawResult[0];
  585. }
  586. //выборка книг автора по authorId
  587. const rows = await this.restoreBooks('author', [authorId]);
  588. let authorName = '';
  589. let books = [];
  590. if (rows.length) {
  591. authorName = rows[0].name;
  592. books = rows[0].books;
  593. }
  594. return {author: authorName, books};
  595. } finally {
  596. this.searchFlag--;
  597. }
  598. }
  599. async getAuthorSeriesList(authorId) {
  600. if (this.closed)
  601. throw new Error('DbSearcher closed');
  602. if (!authorId)
  603. return {author: '', series: []};
  604. this.searchFlag++;
  605. try {
  606. const db = this.db;
  607. //выборка книг автора по authorId
  608. const bookList = await this.getAuthorBookList(authorId);
  609. const books = bookList.books;
  610. const seriesSet = new Set();
  611. for (const book of books) {
  612. if (book.series)
  613. seriesSet.add(book.series.toLowerCase());
  614. }
  615. let series = [];
  616. if (seriesSet.size) {
  617. //выборка серий по названиям
  618. series = await db.select({
  619. table: 'series',
  620. map: `(r) => ({id: r.id, series: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
  621. where: `
  622. const seriesArr = ${db.esc(Array.from(seriesSet))};
  623. const ids = new Set();
  624. for (const value of seriesArr) {
  625. for (const id of @dirtyIndexLR('value', value, value))
  626. ids.add(id);
  627. }
  628. return ids;
  629. `
  630. });
  631. }
  632. return {author: bookList.author, series};
  633. } finally {
  634. this.searchFlag--;
  635. }
  636. }
  637. async getSeriesBookList(series) {
  638. if (this.closed)
  639. throw new Error('DbSearcher closed');
  640. if (!series)
  641. return {books: []};
  642. this.searchFlag++;
  643. try {
  644. const db = this.db;
  645. series = series.toLowerCase();
  646. //выборка серии по названию серии
  647. let rows = await db.select({
  648. table: 'series',
  649. rawResult: true,
  650. where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))`
  651. });
  652. let books = [];
  653. if (rows.length && rows[0].rawResult.length) {
  654. //выборка книг серии
  655. const bookRows = await this.restoreBooks('series', [rows[0].rawResult[0]])
  656. if (bookRows.length)
  657. books = bookRows[0].books;
  658. }
  659. return {books};
  660. } finally {
  661. this.searchFlag--;
  662. }
  663. }
  664. async getCached(key) {
  665. if (!this.queryCacheEnabled)
  666. return null;
  667. let result = null;
  668. const db = this.db;
  669. const memCache = this.memCache;
  670. if (this.queryCacheMemSize > 0 && memCache.has(key)) {//есть в недавних
  671. result = memCache.get(key);
  672. //изменим порядок ключей, для последующей правильной чистки старых
  673. memCache.delete(key);
  674. memCache.set(key, result);
  675. } else if (this.queryCacheDiskSize > 0) {//смотрим в таблице
  676. const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`});
  677. if (rows.length) {//нашли в кеше
  678. await db.insert({
  679. table: 'query_time',
  680. replace: true,
  681. rows: [{id: key, time: Date.now()}],
  682. });
  683. result = rows[0].value;
  684. //заполняем кеш в памяти
  685. if (this.queryCacheMemSize > 0) {
  686. memCache.set(key, result);
  687. if (memCache.size > this.queryCacheMemSize) {
  688. //удаляем самый старый ключ-значение
  689. for (const k of memCache.keys()) {
  690. memCache.delete(k);
  691. break;
  692. }
  693. }
  694. }
  695. }
  696. }
  697. return result;
  698. }
  699. async putCached(key, value) {
  700. if (!this.queryCacheEnabled)
  701. return;
  702. const db = this.db;
  703. if (this.queryCacheMemSize > 0) {
  704. const memCache = this.memCache;
  705. memCache.set(key, value);
  706. if (memCache.size > this.queryCacheMemSize) {
  707. //удаляем самый старый ключ-значение
  708. for (const k of memCache.keys()) {
  709. memCache.delete(k);
  710. break;
  711. }
  712. }
  713. }
  714. if (this.queryCacheDiskSize > 0) {
  715. //кладем в таблицу асинхронно
  716. (async() => {
  717. try {
  718. await db.insert({
  719. table: 'query_cache',
  720. replace: true,
  721. rows: [{id: key, value}],
  722. });
  723. await db.insert({
  724. table: 'query_time',
  725. replace: true,
  726. rows: [{id: key, time: Date.now()}],
  727. });
  728. } catch(e) {
  729. console.error(`putCached: ${e.message}`);
  730. }
  731. })();
  732. }
  733. }
  734. async periodicCleanCache() {
  735. this.timer = null;
  736. const cleanInterval = this.config.cacheCleanInterval*60*1000;
  737. if (!cleanInterval)
  738. return;
  739. try {
  740. if (!this.queryCacheEnabled || this.queryCacheDiskSize <= 0)
  741. return;
  742. const db = this.db;
  743. let rows = await db.select({table: 'query_time', count: true});
  744. const delCount = rows[0].count - this.queryCacheDiskSize;
  745. if (delCount < 1)
  746. return;
  747. //выберем delCount кандидатов на удаление
  748. rows = await db.select({
  749. table: 'query_time',
  750. rawResult: true,
  751. where: `
  752. const delCount = ${delCount};
  753. const rows = [];
  754. @unsafeIter(@all(), (r) => {
  755. rows.push(r);
  756. return false;
  757. });
  758. rows.sort((a, b) => a.time - b.time);
  759. return rows.slice(0, delCount).map(r => r.id);
  760. `
  761. });
  762. const ids = rows[0].rawResult;
  763. //удаляем
  764. await db.delete({table: 'query_cache', where: `@@id(${db.esc(ids)})`});
  765. await db.delete({table: 'query_time', where: `@@id(${db.esc(ids)})`});
  766. //console.log('Cache clean', ids);
  767. } catch(e) {
  768. console.error(e.message);
  769. } finally {
  770. if (!this.closed) {
  771. this.timer = setTimeout(() => { this.periodicCleanCache(); }, cleanInterval);
  772. }
  773. }
  774. }
  775. async close() {
  776. while (this.searchFlag > 0) {
  777. await utils.sleep(50);
  778. }
  779. this.searchCache = null;
  780. if (this.timer) {
  781. clearTimeout(this.timer);
  782. this.timer = null;
  783. }
  784. this.closed = true;
  785. }
  786. }
  787. module.exports = DbSearcher;