f.php 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447
  1. <?php
  2. date_default_timezone_set('Europe/Moscow');
  3. require_once 'config/config.php';
  4. require_once 'parser.php';
  5. define('LOWERCASE',3);
  6. define('UPPERCASE',1);
  7. function getParam($param, $defaultValue = '') {
  8. $paramValue = (isset($_REQUEST[$param]) ? $_REQUEST[$param] : $defaultValue);
  9. return $paramValue;
  10. }
  11. function getEncoding($str, $check_utf = FALSE) {
  12. if (!$check_utf) {
  13. $result = getEncoding(mb_convert_encoding($str, 'cp1251', 'UTF-8'), TRUE);
  14. if ($result == 'w')
  15. return 'u';
  16. }
  17. $charsets = Array(
  18. 'k' => 0,
  19. 'w' => 0,
  20. 'd' => 0,
  21. 'i' => 0,
  22. 'm' => 0
  23. );
  24. $length = strlen($str);
  25. $block_size = ($length > 5*3000) ? 3000 : $length;
  26. $counter = 0;
  27. for ( $i = 0; $i < $length; $i++ ) {
  28. $char = ord($str[$i]);
  29. //non-russian characters
  30. if ($char < 128 || $char > 256)
  31. continue;
  32. //CP866
  33. if (($char > 159 && $char < 176) || ($char > 223 && $char < 242)) $charsets['d']+=LOWERCASE;
  34. if (($char > 127 && $char < 160)) $charsets['d']+=UPPERCASE;
  35. //KOI8-R
  36. if (($char > 191 && $char < 223)) $charsets['k']+=LOWERCASE;
  37. if (($char > 222 && $char < 256)) $charsets['k']+=UPPERCASE;
  38. //WIN-1251
  39. if ($char > 223 && $char < 256) $charsets['w']+=LOWERCASE;
  40. if ($char > 191 && $char < 224) $charsets['w']+=UPPERCASE;
  41. //MAC
  42. if ($char > 221 && $char < 255) $charsets['m']+=LOWERCASE;
  43. if ($char > 127 && $char < 160) $charsets['m']+=UPPERCASE;
  44. //ISO-8859-5
  45. if ($char > 207 && $char < 240) $charsets['i']+=LOWERCASE;
  46. if ($char > 175 && $char < 208) $charsets['i']+=UPPERCASE;
  47. $counter++;
  48. if ($counter > $block_size) {
  49. $counter = 0;
  50. $i += (int)($length/2 - 2*$block_size);
  51. }
  52. }
  53. arsort($charsets);
  54. if (preg_match('//u', $str))
  55. return 'u';
  56. else
  57. return key($charsets);
  58. }
  59. function getTag($tagName, $book) {
  60. $from_tag = '<' . $tagName . '>';
  61. $to_tag = '</' . $tagName . '>';
  62. $from = strpos($book, $from_tag);
  63. $to = strpos($book, $to_tag);
  64. if ($from === FALSE || $to === FALSE)
  65. return '';
  66. $from += strlen($from_tag);
  67. return trim(substr($book, $from, $to - $from));
  68. }
  69. function getMetaInfoAndFilter($book, &$meta_info) {
  70. $meta_info['author'] = '';
  71. $meta_info['title'] = getTag('title', $book);
  72. $out = $book;
  73. //fb2 ??? ---------------------
  74. if (strpos($meta_info['title'], '<p>') !== FALSE) {
  75. $s = str_replace('</p>', '', $meta_info['title']);
  76. $a = explode('<p>', $s);
  77. $meta_info['author'] = parseHtml($a[1], TRUE);
  78. $meta_info['title'] = parseHtml($a[2], TRUE);
  79. if ($meta_info['title'] === NULL || $meta_info['title'] === '') {
  80. $s = parseHtml($s, TRUE);
  81. $meta_info['author'] = '';
  82. $meta_info['title'] = $s;
  83. }
  84. }
  85. //samlib ----------------------
  86. $samlib_start_sign = '<!----------- Ñîáñòâåííî ïðîèçâåäåíèå --------------->';
  87. $samlib_book_idx = strpos($book, $samlib_start_sign);
  88. if ($samlib_book_idx !== FALSE) {
  89. $samlib_author = getTag('h3', $book);
  90. $meta_info['author'] = substr($samlib_author, 0, strpos($samlib_author, ': <small>'));
  91. $meta_info['title'] = getTag('h2', $book);;
  92. $samlib_book_idx += strlen($samlib_start_sign);
  93. $samlib_book_end_idx = strpos($book, '<!---- Áëîê îïèñàíèÿ ïðîèçâåäåíèÿ (ñëåâà âíèçó) ----------------------->');
  94. $samlib_book_end_idx = ($samlib_book_end_idx === FALSE ? strlen($book) : $samlib_book_end_idx);
  95. $out = '<dd>' . $meta_info['author'] . '<dd>' . $meta_info['title'] . '<empty-line/>' .
  96. substr($book, $samlib_book_idx, $samlib_book_end_idx - $samlib_book_idx);
  97. $out = preg_replace("/<dd>&nbsp;&nbsp[;]*\s*[\r\n]/", '<empty-line/>', $out);
  98. }
  99. return $out;
  100. }
  101. function filterTextAndGzip($meta_info, $txtin) {
  102. global $use_gzip;
  103. if (strpos($txtin, '<P>') === FALSE) {
  104. $len = strlen($txtin);
  105. $counts = array();
  106. $flag = 0;
  107. $c = 0;
  108. for ($i = 0; $i < $len; $i++) {
  109. if ($txtin[$i] == chr(10) || $i == 0) {
  110. $counts[$c]++;
  111. if ($c > 0)
  112. $counts[0]++;
  113. $c = 0;
  114. $flag = 1;
  115. } else
  116. if ($txtin[$i] != ' ')
  117. $flag = 0;
  118. else
  119. if ($flag)
  120. $c++;
  121. }
  122. arsort($counts);
  123. $key = 0;
  124. if (count($counts) > 1) {
  125. next($counts);
  126. $key = key($counts);
  127. }
  128. //$txtout .= print_r($counts, TRUE);
  129. //$txtout .= $key;
  130. $txtout = '';
  131. $flag = 0;
  132. $c = 0;
  133. for ($i = 0; $i < $len; $i++) {
  134. if ($txtin[$i] == chr(10) || $i == 0) {
  135. $c = 0;
  136. $flag = 1;
  137. } else
  138. if ($txtin[$i] != ' ') {
  139. if ($c >= $key && $flag)
  140. $txtout .= '<p>';
  141. $flag = 0;
  142. }
  143. else
  144. if ($flag)
  145. $c++;
  146. $txtout .= $txtin[$i];
  147. }
  148. } else
  149. $txtout = $txtin;
  150. $txtout = 'no_file' . '|' . $meta_info['author'] . '|' . $meta_info['title'] .
  151. '<<<bpr5A432688AB0467AA396E5A144830248Abpr>>>' . $txtout;
  152. $supportsGzip = strpos($_SERVER['HTTP_ACCEPT_ENCODING'], 'gzip') !== false;
  153. if ($use_gzip && $supportsGzip && getParam('meta') == '' && getParam('curl') == '') {
  154. $txtout = gzencode($txtout, 9);
  155. header('Content-Encoding: gzip');
  156. }
  157. return $txtout;
  158. }
  159. function myErrorHandler($errno, $errstr, $errfile, $errline)
  160. {
  161. if (!(error_reporting() & $errno)) {
  162. // Ýòîò êîä îøèáêè íå âêëþ÷åí â error_reporting
  163. return;
  164. }
  165. if ($errno == 8 /*|| $errno == 2*/)
  166. return;
  167. //throw new Exception("[$errno]: ($errstr) at $errfile line $errline");
  168. throw new Exception("$errstr");
  169. // Íå çàïóñêàåì âíóòðåííèé îáðàáîò÷èê îøèáîê PHP
  170. return TRUE; // ñþäà õîäà íåò, íî ïóñòü áóäåò êàê øàáëîí
  171. }
  172. function unzip($filein) {
  173. $zip = new ZipArchive;
  174. $result = '';
  175. if ($zip->open($filein) === TRUE) {
  176. $filename = '';
  177. $max_size = -1;
  178. for($i = 0; $i < $zip->numFiles; $i++) {
  179. $stat = $zip->statIndex($i);
  180. $size = $stat['size'];
  181. if ($size > $max_size) {
  182. $max_size = $size;
  183. $filename = $zip->getNameIndex($i);
  184. $fp = $zip->getStream($filename);
  185. if (!$fp)
  186. throw new Exception("zip->getStream failed");
  187. $result = stream_get_contents($fp);
  188. fclose($fp);
  189. }
  190. }
  191. $zip->close();
  192. } else
  193. throw new Exception("zip->open failed");
  194. return $result;
  195. }
  196. function create_guid($namespace = '') {
  197. $uid = md5(uniqid("", true));
  198. $data = $namespace;
  199. $data .= $_SERVER['REQUEST_TIME'];
  200. $data .= $_SERVER['HTTP_USER_AGENT'];
  201. $data .= $_SERVER['LOCAL_ADDR'];
  202. $data .= $_SERVER['LOCAL_PORT'];
  203. $data .= $_SERVER['REMOTE_ADDR'];
  204. $data .= $_SERVER['REMOTE_PORT'];
  205. $hash = strtoupper(hash('ripemd128', $uid . $guid . md5($data)));
  206. return $hash;
  207. }
  208. function microtime_float()
  209. {
  210. list($usec, $sec) = explode(" ", microtime());
  211. return ((float)$usec + (float)$sec);
  212. }
  213. function curlExec(/* Array */$curlOptions='', /* Array */$curlHeaders='', /* Array */$postFields='')
  214. {
  215. $newUrl = '';
  216. $maxRedirection = 10;
  217. do
  218. {
  219. if ($maxRedirection<1) die('Error: reached the limit of redirections');
  220. $ch = curl_init();
  221. curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
  222. curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
  223. if (!empty($curlOptions)) curl_setopt_array($ch, $curlOptions);
  224. if (!empty($curlHeaders)) curl_setopt($ch, CURLOPT_HTTPHEADER, $curlHeaders);
  225. if (!empty($postFields))
  226. {
  227. curl_setopt($ch, CURLOPT_POST, 1);
  228. curl_setopt($ch, CURLOPT_POSTFIELDS, $postFields);
  229. }
  230. if (!empty($newUrl)) curl_setopt($ch, CURLOPT_URL, $newUrl); // redirect needed
  231. curl_setopt($ch, CURLOPT_HEADER, 1);
  232. $response = curl_exec($ch);
  233. // Then, after your curl_exec call:
  234. $header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
  235. $header = substr($response, 0, $header_size);
  236. $curlResult = substr($response, $header_size);
  237. $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
  238. $info = curl_getinfo($ch);
  239. if (getParam('curl') != '') {;
  240. throw new Exception("<br>" . str_replace("[", "<br>[", print_r($info, TRUE)) . "<br>$header<br>END");
  241. }
  242. if ($code == 301 || $code == 302 || $code == 303 || $code == 307)
  243. {
  244. if (array_key_exists('redirect_url', $info) && !empty($info['redirect_url'])) {
  245. $newUrl = trim($info['redirect_url']);
  246. } else {
  247. preg_match('/Location:(.*?)\n/', $header, $matches);
  248. $newUrl = trim(array_pop($matches));
  249. }
  250. curl_close($ch);
  251. $maxRedirection--;
  252. continue;
  253. }
  254. else // no more redirection
  255. {
  256. if ($curlResult === FALSE || $info['http_code'] != 200) {
  257. $curlResult = "ERROR ". $info['http_code'];
  258. if (curl_error($ch))
  259. $curlResult .= "<br>". curl_error($ch);
  260. throw new Exception($curlResult);
  261. } else {
  262. $code = 0; //OK
  263. curl_close($ch);
  264. }
  265. }
  266. }
  267. while($code);
  268. return $curlResult;
  269. }
  270. {
  271. set_error_handler("myErrorHandler");
  272. // set_time_limit(300);
  273. $url = getParam('url');
  274. try {
  275. $body = '';
  276. if ($url == '')
  277. throw new Exception("íå çàäàí àäðåñ êíèãè");
  278. $meta_info = array();
  279. $time_start = $time = microtime_float();
  280. $pid = create_guid();
  281. $dir = 'txt/';
  282. $encoding = getParam('encoding');
  283. if (strpos($url, 'http://') !== 0 && strpos($url, 'https://') !== 0)
  284. $url = 'http://' . $url;
  285. $url = str_replace('"', '', $url);
  286. $url = str_replace('\'', '', $url);
  287. $url = str_replace(']', '%5D', str_replace('[', '%5B', $url));
  288. $options = array(
  289. CURLOPT_RETURNTRANSFER => TRUE,
  290. CURLOPT_TIMEOUT => 300,
  291. CURLOPT_URL => $url,
  292. CURLOPT_BUFFERSIZE => 1024*128,
  293. CURLOPT_NOPROGRESS => FALSE,
  294. CURLOPT_USERAGENT => "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6",
  295. CURLOPT_PROGRESSFUNCTION => function(
  296. $DownloadSize, $Downloaded, $UploadSize, $Uploaded
  297. ) {
  298. // If $Downloaded exceeds, returning non-0 breaks the connection!
  299. return ($Downloaded > (50 * 1024 * 1024)) ? 1 : 0;
  300. }
  301. );
  302. $out = curlExec($options);
  303. $meta_info['time_curl'] = microtime_float() - $time;
  304. $time = microtime_float();
  305. //zip
  306. if ($out[0] == chr(0x50) && $out[1] == chr(0x4B) && $out[2] == chr(0x03) && $out[3] == chr(0x04)) {
  307. $zipped_file = $tmp_dir . "/{$pid}-temp.zip";
  308. file_put_contents($zipped_file, $out);
  309. $out = unzip($zipped_file);
  310. if (file_exists($zipped_file)) unlink($zipped_file);
  311. }
  312. //pdf
  313. /* if ($out[0] == chr(0x25) && $out[1] == chr(0x50) && $out[2] == chr(0x44) && $out[3] == chr(0x46)) {
  314. $a = new PDF2Text();
  315. $a->reset();
  316. $a->decodePDF($out);
  317. $out = $a->output();
  318. file_put_contents('/tmp/1', $out);
  319. }*/
  320. $meta_info['time_unzip'] = microtime_float() - $time;
  321. $time = microtime_float();
  322. //decoding and parsing
  323. if ($out !== FALSE) {
  324. if ($encoding == '')
  325. $encoding = getEncoding($out);
  326. switch ($encoding) {
  327. case 'k':
  328. $out = mb_convert_encoding($out, 'cp1251', 'KOI8-R');
  329. break;
  330. case 'w':
  331. break;
  332. case 'd':
  333. $out = mb_convert_encoding($out, 'cp1251', 'cp866');
  334. break;
  335. case 'i':
  336. $out = mb_convert_encoding($out, 'cp1251', 'ISO-8859-5');
  337. break;
  338. case 'm':
  339. $out = mb_convert_encoding($out, 'cp1251', 'MACINTOSH');
  340. break;
  341. case 'u':
  342. $out = mb_convert_encoding($out, 'cp1251', 'UTF-8');
  343. break;
  344. }
  345. //$out = $encoding . '===' . $out;
  346. //file_put_contents('/tmp/bpr1', $out);
  347. $meta_info['time_decodepage'] = microtime_float() - $time;
  348. $time = microtime_float();
  349. $out = getMetaInfoAndFilter($out, $meta_info);
  350. $meta_info['time_metainfo'] = microtime_float() - $time;
  351. $time = microtime_float();
  352. $out = parseHtml($out);
  353. $meta_info['time_parsehtml'] = microtime_float() - $time;
  354. $time = microtime_float();
  355. $out = filterTextAndGzip($meta_info, $out);
  356. $meta_info['time_filter_gzip'] = microtime_float() - $time;
  357. $meta_info['time_total'] = microtime_float() - $time_start;
  358. $meta = getParam('meta');
  359. if ($meta != '') {
  360. $info = '';
  361. foreach ($meta_info as $key => $value) {
  362. if (strpos($key, 'time') !== FALSE)
  363. $info .= sprintf("%06.3f", $value) . " $key <br>";
  364. else
  365. $info .= "$key: $value<br>";
  366. }
  367. throw new Exception("<br>" . $info);
  368. }
  369. header('Content-Type: text/plain; charset=windows-1251');
  370. echo $out;
  371. //file_put_contents('/tmp/bpr2', $out);
  372. return;
  373. } else
  374. throw new Exception("îøèáêà çàãðóçêè ôàéëà. Ïîïðîáóéòå åùå ðàç.");
  375. } catch (Exception $e) {
  376. header('Content-Type: text/html; charset=windows-1251');
  377. $err = $e->getMessage();
  378. if (strpos($err, 'ERROR 404') !== FALSE)
  379. $err = 'ñòðàíèöà íå íàéäåíà';
  380. $body = "Îøèáêà çàãðóçêè êíèãè: " . ($url == '' ? '' : "($url) ") . $err;
  381. }
  382. echo $body;
  383. }
  384. ?>