index.js 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466
  1. const fs = require('fs-extra');
  2. const URL = require('url').URL;
  3. const iconv = require('iconv-lite');
  4. const chardet = require('chardet');
  5. const _ = require('lodash');
  6. const sax = require('./sax');
  7. const textUtils = require('./textUtils');
  8. const FileDetector = require('../FileDetector');
  9. const repSpaces = (text) => text.replace(/ |[\t\n\r]/g, ' ');
  10. class BookConverter {
  11. constructor() {
  12. this.detector = new FileDetector();
  13. }
  14. async convertToFb2(inputFile, outputFile, url, callback) {
  15. const fileType = await this.detector.detectFile(inputFile);
  16. const data = await fs.readFile(inputFile);
  17. callback(100);
  18. if (fileType && (fileType.ext == 'html' || fileType.ext == 'xml')) {
  19. if (data.toString().indexOf('<FictionBook') >= 0) {
  20. await fs.writeFile(outputFile, this.checkEncoding(data));
  21. return;
  22. }
  23. const parsedUrl = new URL(url);
  24. if (parsedUrl.hostname == 'samlib.ru' ||
  25. parsedUrl.hostname == 'budclub.ru' ||
  26. parsedUrl.hostname == 'zhurnal.lib.ru') {
  27. await fs.writeFile(outputFile, this.convertSamlib(data));
  28. return;
  29. }
  30. await fs.writeFile(outputFile, this.convertHtml(data));
  31. return;
  32. } else {
  33. if (fileType)
  34. throw new Error(`Этот формат файла не поддерживается: ${fileType.mime}`);
  35. else {
  36. //может это чистый текст?
  37. if (textUtils.checkIfText(data)) {
  38. await fs.writeFile(outputFile, this.convertHtml(data, true));
  39. return;
  40. }
  41. throw new Error(`Не удалось определить формат файла: ${url}`);
  42. }
  43. }
  44. }
  45. decode(data) {
  46. const charsetAll = chardet.detectAll(data.slice(0, 20000));
  47. let selected = 'ISO-8859-5';
  48. for (const charset of charsetAll) {
  49. if (charset.name.indexOf('ISO-8859') < 0) {
  50. selected = charset.name;
  51. break;
  52. }
  53. }
  54. if (selected == 'ISO-8859-5') {
  55. selected = textUtils.getEncoding(data);
  56. }
  57. return iconv.decode(data, selected);
  58. }
  59. checkEncoding(data) {
  60. let result = data;
  61. const left = data.indexOf('<?xml version="1.0"');
  62. if (left >= 0) {
  63. const right = data.indexOf('?>', left);
  64. if (right >= 0) {
  65. const head = data.slice(left, right + 2).toString();
  66. const m = head.match(/encoding="(.*)"/);
  67. if (m) {
  68. let encoding = m[1].toLowerCase();
  69. if (encoding != 'utf-8') {
  70. result = iconv.decode(data, encoding);
  71. result = Buffer.from(result.toString().replace(m[0], 'encoding="utf-8"'));
  72. }
  73. }
  74. }
  75. }
  76. return result;
  77. }
  78. convertHtml(data, isText) {
  79. let titleInfo = {};
  80. let desc = {_n: 'description', 'title-info': titleInfo};
  81. let pars = [];
  82. let body = {_n: 'body', section: {_a: []}};
  83. let fb2 = [desc, body];
  84. let title = '';
  85. let inTitle = false;
  86. let spaceCounter = [];
  87. const newParagraph = () => {
  88. pars.push({_n: 'p', _t: ''});
  89. };
  90. const growParagraph = (text) => {
  91. const l = pars.length;
  92. if (l) {
  93. if (pars[l - 1]._t == '')
  94. text = text.trimLeft();
  95. pars[l - 1]._t += text;
  96. }
  97. //посчитаем отступы у текста, чтобы выделить потом параграфы
  98. const lines = text.split('\n');
  99. for (const line of lines) {
  100. const sp = line.split(' ');
  101. let l = 0;
  102. while (l < sp.length && sp[l].trim() == '') {
  103. l++;
  104. }
  105. if (!spaceCounter[l])
  106. spaceCounter[l] = 0;
  107. spaceCounter[l]++;
  108. }
  109. };
  110. newParagraph();
  111. const newPara = new Set(['tr', 'br', 'br/', 'dd', 'p', 'title', '/title', 'h1', 'h2', 'h3', '/h1', '/h2', '/h3']);
  112. const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  113. if (!cutCounter) {
  114. growParagraph(text);
  115. }
  116. if (inTitle && !title)
  117. title = text;
  118. };
  119. const onStartNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  120. if (!cutCounter) {
  121. if (newPara.has(tag))
  122. newParagraph();
  123. }
  124. if (tag == 'title')
  125. inTitle = true;
  126. };
  127. const onEndNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  128. if (tag == 'title')
  129. inTitle = false;
  130. };
  131. let buf = this.decode(data).toString();
  132. sax.parseSync(buf, {
  133. onStartNode, onEndNode, onTextNode,
  134. innerCut: new Set(['head', 'script', 'style'])
  135. });
  136. titleInfo['book-title'] = title;
  137. //подозрение на чистый текст, надо разбить на параграфы
  138. if (isText || pars.length < buf.length/2000) {
  139. let total = 0;
  140. for (let i = 0; i < spaceCounter.length; i++) {
  141. total += (spaceCounter[i] ? spaceCounter[i] : 0);
  142. }
  143. total /= 10;
  144. let i = spaceCounter.length - 1;
  145. while (i > 0 && (!spaceCounter[i] || spaceCounter[i] < total)) i--;
  146. const parIndent = (i > 0 ? i : 0);
  147. let newPars = [];
  148. const newPar = () => {
  149. newPars.push({_n: 'p', _t: ''});
  150. };
  151. const growPar = (text) => {
  152. const l = newPars.length;
  153. if (l) {
  154. newPars[l - 1]._t += text;
  155. }
  156. }
  157. for (const par of pars) {
  158. newPar();
  159. const lines = par._t.split('\n');
  160. for (const line of lines) {
  161. const sp = line.split(' ');
  162. let l = 0;
  163. while (l < sp.length && sp[l].trim() == '') {
  164. l++;
  165. }
  166. if (l >= parIndent)
  167. newPar();
  168. growPar(line.trim() + ' ');
  169. }
  170. }
  171. body.section._a[0] = newPars;
  172. } else {
  173. body.section._a[0] = pars;
  174. }
  175. //убираем лишнее
  176. for (let i = 0; i < pars.length; i++)
  177. pars[i]._t = repSpaces(pars[i]._t).trim();
  178. return this.formatFb2(fb2);
  179. }
  180. convertSamlib(data) {
  181. let titleInfo = {};
  182. let desc = {_n: 'description', 'title-info': titleInfo};
  183. let pars = [];
  184. let body = {_n: 'body', section: {_a: pars}};
  185. let fb2 = [desc, body];
  186. let inSubtitle = false;
  187. let inJustify = true;
  188. let path = '';
  189. let tag = '';// eslint-disable-line no-unused-vars
  190. let inText = false;
  191. let node = {_a: pars};
  192. let inPara = false;
  193. let italic = false;
  194. let bold = false;
  195. const openTag = (name) => {
  196. if (name == 'p')
  197. inPara = true;
  198. let n = {_n: name, _a: [], _p: node};
  199. node._a.push(n);
  200. node = n;
  201. };
  202. const closeTag = (name) => {
  203. if (name == 'p')
  204. inPara = false;
  205. if (node._p) {
  206. const exact = (node._n == name);
  207. node = node._p;
  208. if (!exact)
  209. closeTag(name);
  210. }
  211. };
  212. const growParagraph = (text) => {
  213. if (node._n == 'p' && node._a.length == 0)
  214. text = text.trimLeft();
  215. node._a.push({_t: text});
  216. };
  217. openTag('p');
  218. const onStartNode = (elemName, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  219. if (elemName == '')
  220. return;
  221. if (!inText) {
  222. path += '/' + elemName;
  223. tag = elemName;
  224. } else {
  225. if (inPara && elemName != 'i' && elemName != 'b')
  226. closeTag('p');
  227. switch (elemName) {
  228. case 'li':
  229. case 'p':
  230. case 'dd':
  231. case 'h1':
  232. case 'h2':
  233. case 'h3':
  234. openTag('p');
  235. break;
  236. case 'i':
  237. openTag('emphasis');
  238. italic = true;
  239. break;
  240. case 'b':
  241. openTag('strong');
  242. bold = true;
  243. break;
  244. case 'div':
  245. if (tail.indexOf('align="center"') >= 0) {
  246. openTag('subtitle');
  247. inSubtitle = true;
  248. }
  249. if (tail.indexOf('align="justify"') >= 0) {
  250. openTag('p');
  251. inJustify = true;
  252. }
  253. break;
  254. }
  255. }
  256. };
  257. const onEndNode = (elemName, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  258. if (!inText) {
  259. const oldPath = path;
  260. let t = '';
  261. do {
  262. let i = path.lastIndexOf('/');
  263. t = path.substr(i + 1);
  264. path = path.substr(0, i);
  265. } while (t != elemName && path);
  266. if (t != elemName) {
  267. path = oldPath;
  268. }
  269. let i = path.lastIndexOf('/');
  270. tag = path.substr(i + 1);
  271. } else {
  272. switch (elemName) {
  273. case 'li':
  274. case 'p':
  275. case 'dd':
  276. case 'h1':
  277. case 'h2':
  278. case 'h3':
  279. closeTag('p');
  280. break;
  281. case 'i':
  282. closeTag('emphasis');
  283. italic = false;
  284. break;
  285. case 'b':
  286. closeTag('strong');
  287. bold = false;
  288. break;
  289. case 'div':
  290. if (inSubtitle) {
  291. closeTag('subtitle');
  292. inSubtitle = false;
  293. }
  294. if (inJustify) {
  295. closeTag('p');
  296. inJustify = false;
  297. }
  298. break;
  299. }
  300. }
  301. };
  302. const onComment = (text) => {// eslint-disable-line no-unused-vars
  303. if (text == '--------- Собственно произведение -------------')
  304. inText = true;
  305. if (text == '-----------------------------------------------')
  306. inText = false;
  307. };
  308. const onTextNode = (text) => {// eslint-disable-line no-unused-vars
  309. if (text != ' ' && text.trim() == '')
  310. text = text.trim();
  311. if (text == '')
  312. return;
  313. switch (path) {
  314. case '/html/body/center/h2':
  315. titleInfo['book-title'] = text;
  316. return;
  317. case '/html/body/div/h3':
  318. if (!titleInfo.author)
  319. titleInfo.author = {};
  320. text = text.replace(':', '').trim().split(' ');
  321. if (text[0])
  322. titleInfo.author['last-name'] = text[0];
  323. if (text[1])
  324. titleInfo.author['first-name'] = text[1];
  325. if (text[2])
  326. titleInfo.author['middle-name'] = text[2];
  327. return;
  328. }
  329. let tOpen = (bold ? '<strong>' : '');
  330. tOpen += (italic ? '<emphasis>' : '');
  331. let tClose = (italic ? '</emphasis>' : '');
  332. tClose += (bold ? '</strong>' : '');
  333. if (inText)
  334. growParagraph(`${tOpen}${text}${tClose}`);
  335. };
  336. sax.parseSync(repSpaces(this.decode(data).toString()), {
  337. onStartNode, onEndNode, onTextNode, onComment,
  338. innerCut: new Set(['head', 'script', 'style'])
  339. });
  340. const title = (titleInfo['book-title'] ? titleInfo['book-title'] : '');
  341. let author = '';
  342. if (titleInfo.author) {
  343. author = _.compact([
  344. (titleInfo.author['last-name'] ? titleInfo.author['last-name'] : ''),
  345. (titleInfo.author['first-name'] ? titleInfo.author['first-name'] : ''),
  346. (titleInfo.author['middle-name'] ? titleInfo.author['middle-name'] : ''),
  347. ]).join(' ');
  348. }
  349. pars.unshift({_n: 'title', _a: [
  350. {_n: 'p', _t: author}, {_n: 'p', _t: ''},
  351. {_n: 'p', _t: title}, {_n: 'p', _t: ''},
  352. ]})
  353. return this.formatFb2(fb2);
  354. }
  355. formatFb2(fb2) {
  356. let out = '<?xml version="1.0" encoding="utf-8"?>';
  357. out += '<FictionBook xmlns="http://www.gribuser.ru/xml/fictionbook/2.0" xmlns:l="http://www.w3.org/1999/xlink">';
  358. out += this.formatFb2Node(fb2);
  359. out += '</FictionBook>';
  360. return out;
  361. }
  362. formatFb2Node(node, name) {
  363. let out = '';
  364. if (Array.isArray(node)) {
  365. for (const n of node) {
  366. out += this.formatFb2Node(n);
  367. }
  368. } else if (typeof node == 'string') {
  369. if (name)
  370. out += `<${name}>${repSpaces(node)}</${name}>`;
  371. else
  372. out += repSpaces(node);
  373. } else {
  374. if (node._n)
  375. name = node._n;
  376. if (name)
  377. out += `<${name}>`;
  378. if (node.hasOwnProperty('_t'))
  379. out += repSpaces(node._t);
  380. for (let nodeName in node) {
  381. if (nodeName && nodeName[0] == '_' && nodeName != '_a')
  382. continue;
  383. const n = node[nodeName];
  384. out += this.formatFb2Node(n, nodeName);
  385. }
  386. if (name)
  387. out += `</${name}>`;
  388. }
  389. return out;
  390. }
  391. }
  392. module.exports = BookConverter;