index.js 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500
  1. const fs = require('fs-extra');
  2. const URL = require('url').URL;
  3. const iconv = require('iconv-lite');
  4. const chardet = require('chardet');
  5. const _ = require('lodash');
  6. const sax = require('./sax');
  7. const textUtils = require('./textUtils');
  8. const FileDetector = require('../FileDetector');
  9. const repSpaces = (text) => text.replace(/ |[\t\n\r]/g, ' ');
  10. const repSpaces2 = (text) => text.replace(/[\n\r]/g, '');
  11. class BookConverter {
  12. constructor() {
  13. this.detector = new FileDetector();
  14. }
  15. async convertToFb2(inputFile, outputFile, url, callback) {
  16. const fileType = await this.detector.detectFile(inputFile);
  17. const data = await fs.readFile(inputFile);
  18. callback(100);
  19. if (fileType && (fileType.ext == 'html' || fileType.ext == 'xml')) {
  20. if (data.toString().indexOf('<FictionBook') >= 0) {
  21. await fs.writeFile(outputFile, this.checkEncoding(data));
  22. return;
  23. }
  24. const parsedUrl = new URL(url);
  25. if (parsedUrl.hostname == 'samlib.ru' ||
  26. parsedUrl.hostname == 'budclub.ru' ||
  27. parsedUrl.hostname == 'zhurnal.lib.ru') {
  28. await fs.writeFile(outputFile, this.convertSamlib(data, parsedUrl.hostname));
  29. return;
  30. }
  31. await fs.writeFile(outputFile, this.convertHtml(data));
  32. return;
  33. } else {
  34. if (fileType)
  35. throw new Error(`Этот формат файла не поддерживается: ${fileType.mime}`);
  36. else {
  37. //может это чистый текст?
  38. if (textUtils.checkIfText(data)) {
  39. await fs.writeFile(outputFile, this.convertHtml(data, true));
  40. return;
  41. }
  42. throw new Error(`Не удалось определить формат файла: ${url}`);
  43. }
  44. }
  45. }
  46. decode(data) {
  47. let selected = textUtils.getEncoding(data);
  48. if (selected == 'ISO-8859-5') {
  49. const charsetAll = chardet.detectAll(data.slice(0, 20000));
  50. for (const charset of charsetAll) {
  51. if (charset.name.indexOf('ISO-8859') < 0) {
  52. selected = charset.name;
  53. break;
  54. }
  55. }
  56. }
  57. if (selected.toLowerCase() != 'utf-8')
  58. return iconv.decode(data, selected);
  59. else
  60. return data;
  61. }
  62. checkEncoding(data) {
  63. let result = data;
  64. const left = data.indexOf('<?xml version="1.0"');
  65. if (left >= 0) {
  66. const right = data.indexOf('?>', left);
  67. if (right >= 0) {
  68. const head = data.slice(left, right + 2).toString();
  69. const m = head.match(/encoding="(.*)"/);
  70. if (m) {
  71. let encoding = m[1].toLowerCase();
  72. if (encoding != 'utf-8') {
  73. result = iconv.decode(data, encoding);
  74. result = Buffer.from(result.toString().replace(m[0], 'encoding="utf-8"'));
  75. }
  76. }
  77. }
  78. }
  79. return result;
  80. }
  81. convertHtml(data, isText) {
  82. let titleInfo = {};
  83. let desc = {_n: 'description', 'title-info': titleInfo};
  84. let pars = [];
  85. let body = {_n: 'body', section: {_a: []}};
  86. let fb2 = [desc, body];
  87. let title = '';
  88. let inTitle = false;
  89. let spaceCounter = [];
  90. const newParagraph = () => {
  91. pars.push({_n: 'p', _t: ''});
  92. };
  93. const growParagraph = (text) => {
  94. const l = pars.length;
  95. if (l) {
  96. if (pars[l - 1]._t == '')
  97. text = text.trimLeft();
  98. pars[l - 1]._t += text;
  99. }
  100. //посчитаем отступы у текста, чтобы выделить потом параграфы
  101. const lines = text.split('\n');
  102. for (const line of lines) {
  103. const sp = line.split(' ');
  104. let l = 0;
  105. while (l < sp.length && sp[l].trim() == '') {
  106. l++;
  107. }
  108. if (!spaceCounter[l])
  109. spaceCounter[l] = 0;
  110. spaceCounter[l]++;
  111. }
  112. };
  113. newParagraph();
  114. const newPara = new Set(['tr', 'br', 'br/', 'dd', 'p', 'title', '/title', 'h1', 'h2', 'h3', '/h1', '/h2', '/h3']);
  115. const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  116. if (!cutCounter) {
  117. growParagraph(text);
  118. }
  119. if (inTitle && !title)
  120. title = text;
  121. };
  122. const onStartNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  123. if (!cutCounter) {
  124. if (newPara.has(tag))
  125. newParagraph();
  126. }
  127. if (tag == 'title')
  128. inTitle = true;
  129. };
  130. const onEndNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  131. if (tag == 'title')
  132. inTitle = false;
  133. };
  134. let buf = this.decode(data).toString();
  135. sax.parseSync(buf, {
  136. onStartNode, onEndNode, onTextNode,
  137. innerCut: new Set(['head', 'script', 'style'])
  138. });
  139. titleInfo['book-title'] = title;
  140. //подозрение на чистый текст, надо разбить на параграфы
  141. if (isText || pars.length < buf.length/2000) {
  142. let total = 0;
  143. for (let i = 0; i < spaceCounter.length; i++) {
  144. total += (spaceCounter[i] ? spaceCounter[i] : 0);
  145. }
  146. total /= 10;
  147. let i = spaceCounter.length - 1;
  148. while (i > 0 && (!spaceCounter[i] || spaceCounter[i] < total)) i--;
  149. const parIndent = (i > 0 ? i : 0);
  150. let newPars = [];
  151. const newPar = () => {
  152. newPars.push({_n: 'p', _t: ''});
  153. };
  154. const growPar = (text) => {
  155. const l = newPars.length;
  156. if (l) {
  157. newPars[l - 1]._t += text;
  158. }
  159. }
  160. for (const par of pars) {
  161. newPar();
  162. const lines = par._t.split('\n');
  163. for (const line of lines) {
  164. const sp = line.split(' ');
  165. let l = 0;
  166. while (l < sp.length && sp[l].trim() == '') {
  167. l++;
  168. }
  169. if (l >= parIndent)
  170. newPar();
  171. growPar(line.trim() + ' ');
  172. }
  173. }
  174. body.section._a[0] = newPars;
  175. } else {
  176. body.section._a[0] = pars;
  177. }
  178. //убираем лишнее
  179. for (let i = 0; i < pars.length; i++)
  180. pars[i]._t = repSpaces(pars[i]._t).trim();
  181. return this.formatFb2(fb2);
  182. }
  183. convertSamlib(data, hostname) {
  184. let titleInfo = {};
  185. let desc = {_n: 'description', 'title-info': titleInfo};
  186. let pars = [];
  187. let body = {_n: 'body', section: {_a: pars}};
  188. let fb2 = [desc, body];
  189. let inSubtitle = false;
  190. let inJustify = true;
  191. let inImage = false;
  192. let path = '';
  193. let tag = '';// eslint-disable-line no-unused-vars
  194. let inText = false;
  195. let textFound = false;
  196. let node = {_a: pars};
  197. let inPara = false;
  198. let italic = false;
  199. let bold = false;
  200. const openTag = (name, attrs) => {
  201. if (name == 'p')
  202. inPara = true;
  203. let n = {_n: name, _attrs: attrs, _a: [], _p: node};
  204. node._a.push(n);
  205. node = n;
  206. };
  207. const closeTag = (name) => {
  208. if (name == 'p')
  209. inPara = false;
  210. if (node._p) {
  211. const exact = (node._n == name);
  212. node = node._p;
  213. if (!exact)
  214. closeTag(name);
  215. }
  216. };
  217. const growParagraph = (text) => {
  218. if (node._n == 'p' && node._a.length == 0)
  219. text = text.trimLeft();
  220. node._a.push({_t: text});
  221. };
  222. openTag('p');
  223. const onStartNode = (elemName, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  224. if (elemName == '')
  225. return;
  226. if (!inText) {
  227. path += '/' + elemName;
  228. tag = elemName;
  229. } else {
  230. if (inPara && elemName != 'i' && elemName != 'b' && elemName != 'em' && elemName != 'strong' && elemName != 'img')
  231. closeTag('p');
  232. switch (elemName) {
  233. case 'li':
  234. case 'p':
  235. case 'dd':
  236. case 'h1':
  237. case 'h2':
  238. case 'h3':
  239. case 'br':
  240. openTag('p');
  241. break;
  242. case 'i':
  243. case 'em':
  244. italic = true;
  245. break;
  246. case 'b':
  247. case 'strong':
  248. bold = true;
  249. break;
  250. case 'div':
  251. if (tail.indexOf('align="center"') >= 0) {
  252. openTag('subtitle');
  253. inSubtitle = true;
  254. }
  255. if (tail.indexOf('align="justify"') >= 0) {
  256. openTag('p');
  257. inJustify = true;
  258. }
  259. break;
  260. case 'img': {
  261. const attrs = sax.getAttrsSync(tail);
  262. if (attrs.src && attrs.src.value) {
  263. let href = attrs.src.value;
  264. if (href[0] == '/')
  265. href = `http://${hostname}${href}`;
  266. openTag('image', {href});
  267. inImage = true;
  268. }
  269. break;
  270. }
  271. }
  272. }
  273. };
  274. const onEndNode = (elemName, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
  275. if (!inText) {
  276. const oldPath = path;
  277. let t = '';
  278. do {
  279. let i = path.lastIndexOf('/');
  280. t = path.substr(i + 1);
  281. path = path.substr(0, i);
  282. } while (t != elemName && path);
  283. if (t != elemName) {
  284. path = oldPath;
  285. }
  286. let i = path.lastIndexOf('/');
  287. tag = path.substr(i + 1);
  288. } else {
  289. switch (elemName) {
  290. case 'li':
  291. case 'p':
  292. case 'dd':
  293. case 'h1':
  294. case 'h2':
  295. case 'h3':
  296. closeTag('p');
  297. break;
  298. case 'i':
  299. case 'em':
  300. italic = false;
  301. break;
  302. case 'b':
  303. case 'strong':
  304. bold = false;
  305. break;
  306. case 'div':
  307. if (inSubtitle) {
  308. closeTag('subtitle');
  309. inSubtitle = false;
  310. }
  311. if (inJustify) {
  312. closeTag('p');
  313. inJustify = false;
  314. }
  315. break;
  316. case 'img':
  317. if (inImage)
  318. closeTag('image');
  319. inImage = false;
  320. break;
  321. }
  322. }
  323. };
  324. const onComment = (text) => {// eslint-disable-line no-unused-vars
  325. if (text == '--------- Собственно произведение -------------') {
  326. inText = true;
  327. textFound = true;
  328. }
  329. if (text == '-----------------------------------------------')
  330. inText = false;
  331. };
  332. const onTextNode = (text) => {// eslint-disable-line no-unused-vars
  333. if (text != ' ' && text.trim() == '')
  334. text = text.trim();
  335. if (text == '')
  336. return;
  337. switch (path) {
  338. case '/html/body/center/h2':
  339. titleInfo['book-title'] = text;
  340. return;
  341. case '/html/body/div/h3':
  342. if (!titleInfo.author)
  343. titleInfo.author = {};
  344. text = text.replace(':', '').trim().split(' ');
  345. if (text[0])
  346. titleInfo.author['last-name'] = text[0];
  347. if (text[1])
  348. titleInfo.author['first-name'] = text[1];
  349. if (text[2])
  350. titleInfo.author['middle-name'] = text[2];
  351. return;
  352. }
  353. let tOpen = (bold ? '<strong>' : '');
  354. tOpen += (italic ? '<emphasis>' : '');
  355. let tClose = (italic ? '</emphasis>' : '');
  356. tClose += (bold ? '</strong>' : '');
  357. if (inText)
  358. growParagraph(`${tOpen}${text}${tClose}`);
  359. };
  360. sax.parseSync(repSpaces(repSpaces2(this.decode(data).toString())), {
  361. onStartNode, onEndNode, onTextNode, onComment,
  362. innerCut: new Set(['head', 'script', 'style'])
  363. });
  364. //текст не найден на странице, обрабатываем как html
  365. if (!textFound)
  366. return this.convertHtml(data);
  367. const title = (titleInfo['book-title'] ? titleInfo['book-title'] : '');
  368. let author = '';
  369. if (titleInfo.author) {
  370. author = _.compact([
  371. (titleInfo.author['last-name'] ? titleInfo.author['last-name'] : ''),
  372. (titleInfo.author['first-name'] ? titleInfo.author['first-name'] : ''),
  373. (titleInfo.author['middle-name'] ? titleInfo.author['middle-name'] : ''),
  374. ]).join(' ');
  375. }
  376. pars.unshift({_n: 'title', _a: [
  377. {_n: 'p', _t: author}, {_n: 'p', _t: ''},
  378. {_n: 'p', _t: title}, {_n: 'p', _t: ''},
  379. ]})
  380. return this.formatFb2(fb2);
  381. }
  382. formatFb2(fb2) {
  383. let out = '<?xml version="1.0" encoding="utf-8"?>';
  384. out += '<FictionBook xmlns="http://www.gribuser.ru/xml/fictionbook/2.0" xmlns:l="http://www.w3.org/1999/xlink">';
  385. out += this.formatFb2Node(fb2);
  386. out += '</FictionBook>';
  387. return out;
  388. }
  389. formatFb2Node(node, name) {
  390. let out = '';
  391. if (Array.isArray(node)) {
  392. for (const n of node) {
  393. out += this.formatFb2Node(n);
  394. }
  395. } else if (typeof node == 'string') {
  396. if (name)
  397. out += `<${name}>${repSpaces(node)}</${name}>`;
  398. else
  399. out += repSpaces(node);
  400. } else {
  401. if (node._n)
  402. name = node._n;
  403. let attrs = '';
  404. if (node._attrs) {
  405. for (let attrName in node._attrs) {
  406. attrs += ` ${attrName}="${node._attrs[attrName]}"`;
  407. }
  408. }
  409. if (name)
  410. out += `<${name}${attrs}>`;
  411. if (node.hasOwnProperty('_t'))
  412. out += repSpaces(node._t);
  413. for (let nodeName in node) {
  414. if (nodeName && nodeName[0] == '_' && nodeName != '_a')
  415. continue;
  416. const n = node[nodeName];
  417. out += this.formatFb2Node(n, nodeName);
  418. }
  419. if (name)
  420. out += `</${name}>`;
  421. }
  422. return out;
  423. }
  424. }
  425. module.exports = BookConverter;