You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

179 lines
5.2 KiB

8 years ago
  1. <?php
  2. use Symfony\Component\HttpFoundation\Request;
  3. use Symfony\Component\HttpFoundation\Response;
  4. use XRay\Formats;
  5. class Parse {
  6. public $http;
  7. public $mc;
  8. private $_cacheTime = 300;
  9. private $_pretty = false;
  10. public function __construct() {
  11. $this->http = new p3k\HTTP();
  12. if(class_exists('Memcache')) {
  13. $this->mc = new Memcache();
  14. $this->mc->addServer('127.0.0.1');
  15. }
  16. }
  17. public static function debug($msg, $header='X-Parse-Debug') {
  18. syslog(LOG_INFO, $msg);
  19. if(array_key_exists('REMOTE_ADDR', $_SERVER))
  20. header($header . ": " . $msg);
  21. }
  22. private function respond(Response $response, $code, $params, $headers=[]) {
  23. $response->setStatusCode($code);
  24. foreach($headers as $k=>$v) {
  25. $response->headers->set($k, $v);
  26. }
  27. $response->headers->set('Content-Type', 'application/json');
  28. $opts = JSON_UNESCAPED_SLASHES;
  29. if($this->_pretty) $opts += JSON_PRETTY_PRINT;
  30. $response->setContent(json_encode($params, $opts)."\n");
  31. return $response;
  32. }
  33. private static function toHtmlEntities($input) {
  34. return mb_convert_encoding($input, 'HTML-ENTITIES', mb_detect_encoding($input));
  35. }
  36. public function parse(Request $request, Response $response) {
  37. if($request->get('timeout')) {
  38. // We might make 2 HTTP requests, so each request gets half the desired timeout
  39. $this->http->timeout = $request->get('timeout') / 2;
  40. }
  41. if($request->get('max_redirects')) {
  42. $this->http->max_redirects = (int)$request->get('max_redirects');
  43. }
  44. if($request->get('pretty')) {
  45. $this->_pretty = true;
  46. }
  47. $url = $request->get('url');
  48. $html = $request->get('html');
  49. if(!$url && !$html) {
  50. return $this->respond($response, 400, [
  51. 'error' => 'missing_url',
  52. 'error_description' => 'Provide a URL or HTML to fetch'
  53. ]);
  54. }
  55. if($html) {
  56. // If HTML is provided in the request, parse that, and use the URL provided as the base URL for mf2 resolving
  57. $result['body'] = $html;
  58. } else {
  59. // Attempt some basic URL validation
  60. $scheme = parse_url($url, PHP_URL_SCHEME);
  61. if(!in_array($scheme, ['http','https'])) {
  62. return $this->respond($response, 400, [
  63. 'error' => 'invalid_url',
  64. 'error_description' => 'Only http and https URLs are supported'
  65. ]);
  66. }
  67. $host = parse_url($url, PHP_URL_HOST);
  68. if(!$host) {
  69. return $this->respond($response, 400, [
  70. 'error' => 'invalid_url',
  71. 'error_description' => 'The URL provided was not valid'
  72. ]);
  73. }
  74. $url = \normalize_url($url);
  75. // Now fetch the URL and check for any curl errors
  76. if($this->mc) {
  77. $cacheKey = 'xray-'.md5($url);
  78. if($cached=$this->mc->get($cacheKey)) {
  79. $result = json_decode($cached, true);
  80. self::debug('using HTML from cache', 'X-Cache-Debug');
  81. } else {
  82. $result = $this->http->get($url);
  83. $cacheData = json_encode($result);
  84. // App Engine limits the size of cached items, so don't cache ones larger than that
  85. if(strlen($cacheData) < 1000000)
  86. $this->mc->set($cacheKey, $cacheData, MEMCACHE_COMPRESSED, $this->_cacheTime);
  87. }
  88. } else {
  89. $result = $this->http->get($url);
  90. }
  91. if($result['error']) {
  92. return $this->respond($response, 400, [
  93. 'error' => $result['error'],
  94. 'error_description' => $result['error_description']
  95. ]);
  96. }
  97. if(trim($result['body']) == '') {
  98. return $this->respond($response, 200, [
  99. 'error' => 'no_content',
  100. 'error_description' => 'We did not get a response body when fetching the URL'
  101. ]);
  102. }
  103. }
  104. // attempt to parse the page as HTML
  105. $doc = new DOMDocument();
  106. @$doc->loadHTML(self::toHtmlEntities($result['body']));
  107. if(!$doc) {
  108. return $this->respond($response, 400, [
  109. 'error' => 'invalid_content',
  110. 'error_description' => 'The document could not be parsed as HTML'
  111. ]);
  112. }
  113. // If a target parameter was provided, make sure a link to it exists on the page
  114. if($target=$request->get('target')) {
  115. $xpath = new DOMXPath($doc);
  116. $found = [];
  117. foreach($xpath->query('//a[@href]') as $href) {
  118. $u = $href->getAttribute('href');
  119. if($target) {
  120. # target parameter was provided
  121. if($u == $target) {
  122. $found[$u] = null;
  123. }
  124. }
  125. }
  126. if(!$found) {
  127. return $this->respond($response, 400, [
  128. 'error' => 'no_link_found',
  129. 'error_description' => 'The source document does not have a link to the target URL'
  130. ]);
  131. }
  132. }
  133. // Now start pulling in the data from the page. Start by looking for microformats2
  134. $mf2 = mf2\Parse($result['body'], $result['url']);
  135. if($mf2 && count($mf2['items']) > 0) {
  136. $data = Formats\Mf2::parse($mf2, $result['url'], $this->http);
  137. if($data) {
  138. return $this->respond($response, 200, $data);
  139. }
  140. }
  141. // TODO: look for other content like OEmbed or other known services later
  142. return $this->respond($response, 200, [
  143. 'data' => [
  144. 'type' => 'unknown',
  145. ]
  146. ]);
  147. }
  148. }