inpx-web/server/core/DbCreator.js

const InpxParser = require('./InpxParser');
const utils = require('./utils');

class DbCreator {
    constructor(config) {
        this.config = config;
    }

    async run(db, callback) {
        const config = this.config;

        //book
        await db.create({
            table: 'book'
        });

        callback({job: 'load inpx', jobMessage: 'Загрузка INPX'});
        const readFileCallback = async(readState) => {
            callback(readState);
        };

        //поисковые таблицы, ниже сохраним в БД
        let authorMap = new Map();//авторы
        let authorArr = [];
        let seriesMap = new Map();//серии
        let seriesArr = [];
        let titleMap = new Map();//названия
        let titleArr = [];
        let genreMap = new Map();//жанры
        let genreArr = [];
        let langMap = new Map();//языки
        let langArr = [];

        //stats
        let authorCount = 0;
        let bookCount = 0;
        let noAuthorBookCount = 0;
        let bookDelCount = 0;

        //stuff
        let recsLoaded = 0;
        let id = 0;
        let chunkNum = 0;
        const parsedCallback = async(chunk) => {
            for (const rec of chunk) {
                rec.id = ++id;

                if (!rec.del)
                    bookCount++;
                else 
                    bookDelCount++;

                if (!rec.author) {
                    if (!rec.del)
                        noAuthorBookCount++;
                    rec.author = 'Автор не указан';
                }

                //авторы
                const author = rec.author.split(',');
                if (author.length > 1)
                    author.push(rec.author);

                const authorIds = [];
                for (let i = 0; i < author.length; i++) {
                    const a = author[i];

                    let authorRec;                    
                    if (authorMap.has(a)) {
                         const authorId = authorMap.get(a);
                         authorRec = authorArr[authorId];
                    } else {
                        authorRec = {id: authorArr.length, author: a, value: a.toLowerCase(), bookId: []};
                        authorArr.push(authorRec);
                        authorMap.set(a, authorRec.id);

                        if (author.length == 1 || i < author.length - 1) //без соавторов
                            authorCount++;
                    }

                    authorRec.bookId.push(id);
                    authorIds.push(authorRec.id);
                }

                //серии
                if (rec.series) {
                    const series = rec.series;

                    let seriesRec;
                    if (seriesMap.has(series)) {
                        const seriesId = seriesMap.get(series);
                        seriesRec = seriesArr[seriesId];
                    } else {
                        seriesRec = {id: seriesArr.length, value: series.toLowerCase(), authorId: new Set()};
                        seriesArr.push(seriesRec);
                        seriesMap.set(series, seriesRec.id);
                    }

                    for (const id of authorIds) {
                        seriesRec.authorId.add(id);
                    }
                }

                //названия
                if (rec.title) {
                    const title = rec.title;

                    let titleRec;
                    if (titleMap.has(title)) {
                        const titleId = titleMap.get(title);
                        titleRec = titleArr[titleId];
                    } else {
                        titleRec = {id: titleArr.length, value: title.toLowerCase(), authorId: new Set()};
                        titleArr.push(titleRec);
                        titleMap.set(title, titleRec.id);
                    }

                    for (const id of authorIds) {
                        titleRec.authorId.add(id);
                    }
                }

                //жанры
                if (rec.genre) {
                    const genre = rec.genre.split(',');

                    for (const g of genre) {
                        let genreRec;
                        if (genreMap.has(g)) {
                            const genreId = genreMap.get(g);
                            genreRec = genreArr[genreId];
                        } else {
                            genreRec = {id: genreArr.length, value: g, authorId: new Set()};
                            genreArr.push(genreRec);
                            genreMap.set(g, genreRec.id);
                        }

                        for (const id of authorIds) {
                            genreRec.authorId.add(id);
                        }
                    }
                }

                //языки
                if (rec.lang) {
                    const lang = rec.lang;

                    let langRec;
                    if (langMap.has(lang)) {
                        const langId = langMap.get(lang);
                        langRec = langArr[langId];
                    } else {
                        langRec = {id: langArr.length, value: lang, authorId: new Set()};
                        langArr.push(langRec);
                        langMap.set(lang, langRec.id);
                    }

                    for (const id of authorIds) {
                        langRec.authorId.add(id);
                    }
                }
            }

            await db.insert({table: 'book', rows: chunk});
            
            recsLoaded += chunk.length;
            callback({recsLoaded});

            if (chunkNum++ % 10 == 0)
                utils.freeMemory();
        };

        //парсинг
        const parser = new InpxParser();
        await parser.parse(config.inpxFile, readFileCallback, parsedCallback);

        //чистка памяти, ибо жрет как не в себя
        authorMap = null;
        seriesMap = null;
        titleMap = null;
        genreMap = null;

        utils.freeMemory();

        //config
        callback({job: 'config save', jobMessage: 'Сохранение конфигурации'});
        await db.create({
            table: 'config'
        });

        const stats = {
            recsLoaded,
            authorCount,
            authorCountAll: authorArr.length,
            bookCount,
            bookCountAll: bookCount + bookDelCount,
            bookDelCount,
            noAuthorBookCount,
            titleCount: titleArr.length,
            seriesCount: seriesArr.length,
            genreCount: genreArr.length,
            langCount: langArr.length,
        };
        //console.log(stats);

        const inpxHash = await utils.getFileHash(config.inpxFile, 'sha256', 'hex');

        await db.insert({table: 'config', rows: [
            {id: 'inpxInfo', value: parser.info},
            {id: 'stats', value: stats},
            {id: 'inpxHash', value: inpxHash},
        ]});

        //сохраним поисковые таблицы
        const chunkSize = 10000;

        //author
        callback({job: 'author save', jobMessage: 'Сохранение авторов книг'});
        await db.create({
            table: 'author',
            index: {field: 'value', depth: config.indexDepth},
        });

        //вставка в БД по кусочкам, экономим память
        for (let i = 0; i < authorArr.length; i += chunkSize) {
            const chunk = authorArr.slice(i, i + chunkSize);

            await db.insert({table: 'author', rows: chunk});
        }

        authorArr = null;
        await db.close({table: 'author'});
        utils.freeMemory();

        //series
        callback({job: 'series save', jobMessage: 'Сохранение серий книг'});
        await db.create({
            table: 'series',
            index: {field: 'value', depth: config.indexDepth},
        });

        //вставка в БД по кусочкам, экономим память
        for (let i = 0; i < seriesArr.length; i += chunkSize) {
            const chunk = seriesArr.slice(i, i + chunkSize);
            for (const rec of chunk)
                rec.authorId = Array.from(rec.authorId);

            await db.insert({table: 'series', rows: chunk});
        }

        seriesArr = null;
        await db.close({table: 'series'});
        utils.freeMemory();

        //title
        callback({job: 'title save', jobMessage: 'Сохранение названий книг'});
        await db.create({
            table: 'title',
            index: {field: 'value', depth: config.indexDepth},
        });

        //вставка в БД по кусочкам, экономим память
        let j = 0;
        for (let i = 0; i < titleArr.length; i += chunkSize) {
            const chunk = titleArr.slice(i, i + chunkSize);
            for (const rec of chunk)
                rec.authorId = Array.from(rec.authorId);

            await db.insert({table: 'title', rows: chunk});
            if (j++ % 10 == 0)
                utils.freeMemory();
            await utils.sleep(100);
        }

        titleArr = null;
        await db.close({table: 'title'});
        utils.freeMemory();

        //genre
        callback({job: 'genre save', jobMessage: 'Сохранение жанров'});
        await db.create({
            table: 'genre',
            index: {field: 'value', depth: config.indexDepth},
        });

        await db.insert({table: 'genre', rows: genreArr});

        genreArr = null;
        await db.close({table: 'genre'});
        utils.freeMemory();

        //lang
        callback({job: 'lang save', jobMessage: 'Сохранение языков'});
        await db.create({
            table: 'lang',
            index: {field: 'value', depth: config.indexDepth},
        });

        await db.insert({table: 'lang', rows: langArr});

        langArr = null;
        await db.close({table: 'lang'});
        utils.freeMemory();

        callback({job: 'done', jobMessage: ''});
    }
}

module.exports = DbCreator;