You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

350 lines
11 KiB

8 years ago
  1. <?php
  2. use Symfony\Component\HttpFoundation\Request;
  3. use Symfony\Component\HttpFoundation\Response;
  4. use XRay\Formats;
  5. class Parse {
  6. public $http;
  7. public $mc;
  8. private $_cacheTime = 120;
  9. private $_pretty = false;
  10. public function __construct() {
  11. $this->http = new p3k\HTTP();
  12. if(Config::$cache && class_exists('Memcache')) {
  13. $this->mc = new Memcache();
  14. $this->mc->addServer('127.0.0.1');
  15. }
  16. }
  17. public static function debug($msg, $header='X-Parse-Debug') {
  18. syslog(LOG_INFO, $msg);
  19. if(array_key_exists('REMOTE_ADDR', $_SERVER))
  20. header($header . ": " . $msg);
  21. }
  22. private function respond(Response $response, $code, $params, $headers=[]) {
  23. $response->setStatusCode($code);
  24. foreach($headers as $k=>$v) {
  25. $response->headers->set($k, $v);
  26. }
  27. $response->headers->set('Content-Type', 'application/json');
  28. $opts = JSON_UNESCAPED_SLASHES;
  29. if($this->_pretty) $opts += JSON_PRETTY_PRINT;
  30. $response->setContent(json_encode($params, $opts)."\n");
  31. return $response;
  32. }
  33. private static function toHtmlEntities($input) {
  34. return mb_convert_encoding($input, 'HTML-ENTITIES', mb_detect_encoding($input));
  35. }
  36. public function parse(Request $request, Response $response) {
  37. if($request->get('timeout')) {
  38. // We might make 2 HTTP requests, so each request gets half the desired timeout
  39. $this->http->timeout = $request->get('timeout') / 2;
  40. }
  41. if($request->get('max_redirects')) {
  42. $this->http->max_redirects = (int)$request->get('max_redirects');
  43. }
  44. if($request->get('pretty')) {
  45. $this->_pretty = true;
  46. }
  47. $url = $request->get('url');
  48. $html = $request->get('html');
  49. if(!$url && !$html) {
  50. return $this->respond($response, 400, [
  51. 'error' => 'missing_url',
  52. 'error_description' => 'Provide a URL or HTML to fetch'
  53. ]);
  54. }
  55. if($html) {
  56. // If HTML is provided in the request, parse that, and use the URL provided as the base URL for mf2 resolving
  57. $result['body'] = $html;
  58. $result['url'] = $url;
  59. } else {
  60. // Attempt some basic URL validation
  61. $scheme = parse_url($url, PHP_URL_SCHEME);
  62. if(!in_array($scheme, ['http','https'])) {
  63. return $this->respond($response, 400, [
  64. 'error' => 'invalid_url',
  65. 'error_description' => 'Only http and https URLs are supported'
  66. ]);
  67. }
  68. $host = parse_url($url, PHP_URL_HOST);
  69. if(!$host) {
  70. return $this->respond($response, 400, [
  71. 'error' => 'invalid_url',
  72. 'error_description' => 'The URL provided was not valid'
  73. ]);
  74. }
  75. $url = \normalize_url($url);
  76. // Check if this is a Twitter URL and if they've provided API credentials, use the API
  77. if(preg_match('/https?:\/\/(?:mobile\.twitter\.com|twitter\.com|twtr\.io)\/(?:[a-z0-9_\/!#]+statuse?s?\/([0-9]+)|([a-zA-Z0-9_]+))/i', $url, $match)) {
  78. return $this->parseTwitterURL($request, $response, $url, $match);
  79. }
  80. // Now fetch the URL and check for any curl errors
  81. // Don't cache the response if a token is used to fetch it
  82. if($this->mc && !$request->get('token')) {
  83. $cacheKey = 'xray-'.md5($url);
  84. if($cached=$this->mc->get($cacheKey)) {
  85. $result = json_decode($cached, true);
  86. self::debug('using HTML from cache', 'X-Cache-Debug');
  87. } else {
  88. $result = $this->http->get($url);
  89. $cacheData = json_encode($result);
  90. // App Engine limits the size of cached items, so don't cache ones larger than that
  91. if(strlen($cacheData) < 1000000)
  92. $this->mc->set($cacheKey, $cacheData, MEMCACHE_COMPRESSED, $this->_cacheTime);
  93. }
  94. } else {
  95. $headers = [];
  96. if($request->get('token')) {
  97. $headers[] = 'Authorization: Bearer ' . $request->get('token');
  98. }
  99. $result = $this->http->get($url, $headers);
  100. }
  101. if($result['error']) {
  102. return $this->respond($response, 200, [
  103. 'error' => $result['error'],
  104. 'error_description' => $result['error_description'],
  105. 'url' => $result['url'],
  106. 'code' => $result['code']
  107. ]);
  108. }
  109. if(trim($result['body']) == '') {
  110. if($result['code'] == 410) {
  111. // 410 Gone responses are valid and should not return an error
  112. return $this->respond($response, 200, [
  113. 'data' => [
  114. 'type' => 'unknown'
  115. ],
  116. 'url' => $result['url'],
  117. 'code' => $result['code']
  118. ]);
  119. }
  120. return $this->respond($response, 200, [
  121. 'error' => 'no_content',
  122. 'error_description' => 'We did not get a response body when fetching the URL',
  123. 'url' => $result['url'],
  124. 'code' => $result['code']
  125. ]);
  126. }
  127. // Check for HTTP 401/403
  128. if($result['code'] == 401) {
  129. return $this->respond($response, 200, [
  130. 'error' => 'unauthorized',
  131. 'error_description' => 'The URL returned "HTTP 401 Unauthorized"',
  132. 'url' => $result['url'],
  133. 'code' => 401
  134. ]);
  135. }
  136. if($result['code'] == 403) {
  137. return $this->respond($response, 200, [
  138. 'error' => 'forbidden',
  139. 'error_description' => 'The URL returned "HTTP 403 Forbidden"',
  140. 'url' => $result['url'],
  141. 'code' => 403
  142. ]);
  143. }
  144. }
  145. // Check for known services
  146. $host = parse_url($result['url'], PHP_URL_HOST);
  147. if(in_array($host, ['www.instagram.com','instagram.com'])) {
  148. list($data, $parsed) = Formats\Instagram::parse($result['body'], $result['url'], $this->http);
  149. if($request->get('include_original'))
  150. $data['original'] = $parsed;
  151. $data['url'] = $result['url'];
  152. $data['code'] = $result['code'];
  153. return $this->respond($response, 200, $data);
  154. }
  155. if($host == 'xkcd.com' && parse_url($url, PHP_URL_PATH) != '/') {
  156. $data = Formats\XKCD::parse($result['body'], $url);
  157. $data['url'] = $result['url'];
  158. $data['code'] = $result['code'];
  159. return $this->respond($response, 200, $data);
  160. }
  161. // attempt to parse the page as HTML
  162. $doc = new DOMDocument();
  163. @$doc->loadHTML(self::toHtmlEntities($result['body']));
  164. if(!$doc) {
  165. return $this->respond($response, 200, [
  166. 'error' => 'invalid_content',
  167. 'error_description' => 'The document could not be parsed as HTML'
  168. ]);
  169. }
  170. $xpath = new DOMXPath($doc);
  171. // Check for meta http equiv and replace the status code if present
  172. foreach($xpath->query('//meta[translate(@http-equiv,\'STATUS\',\'status\')=\'status\']') as $el) {
  173. $equivStatus = ''.$el->getAttribute('content');
  174. if($equivStatus && is_string($equivStatus)) {
  175. if(preg_match('/^(\d+)/', $equivStatus, $match)) {
  176. $result['code'] = (int)$match[1];
  177. }
  178. }
  179. }
  180. // If a target parameter was provided, make sure a link to it exists on the page
  181. if($target=$request->get('target')) {
  182. $found = [];
  183. if($target) {
  184. self::xPathFindNodeWithAttribute($xpath, 'a', 'href', function($u) use($target, &$found){
  185. if($u == $target) {
  186. $found[$u] = null;
  187. }
  188. });
  189. self::xPathFindNodeWithAttribute($xpath, 'img', 'src', function($u) use($target, &$found){
  190. if($u == $target) {
  191. $found[$u] = null;
  192. }
  193. });
  194. self::xPathFindNodeWithAttribute($xpath, 'video', 'src', function($u) use($target, &$found){
  195. if($u == $target) {
  196. $found[$u] = null;
  197. }
  198. });
  199. self::xPathFindNodeWithAttribute($xpath, 'audio', 'src', function($u) use($target, &$found){
  200. if($u == $target) {
  201. $found[$u] = null;
  202. }
  203. });
  204. }
  205. if(!$found) {
  206. return $this->respond($response, 200, [
  207. 'error' => 'no_link_found',
  208. 'error_description' => 'The source document does not have a link to the target URL',
  209. 'url' => $result['url'],
  210. 'code' => $result['code'],
  211. ]);
  212. }
  213. }
  214. // If the URL has a fragment ID, find the DOM starting at that node and parse it instead
  215. $html = $result['body'];
  216. $fragment = parse_url($url, PHP_URL_FRAGMENT);
  217. if($fragment) {
  218. $fragElement = self::xPathGetElementById($xpath, $fragment);
  219. if($fragElement) {
  220. $html = $doc->saveHTML($fragElement);
  221. $foundFragment = true;
  222. } else {
  223. $foundFragment = false;
  224. }
  225. }
  226. // Now start pulling in the data from the page. Start by looking for microformats2
  227. $mf2 = mf2\Parse($html, $result['url']);
  228. if($mf2 && count($mf2['items']) > 0) {
  229. $data = Formats\Mf2::parse($mf2, $result['url'], $this->http);
  230. if($data) {
  231. if($fragment) {
  232. $data['info'] = [
  233. 'found_fragment' => $foundFragment
  234. ];
  235. }
  236. if($request->get('include_original'))
  237. $data['original'] = $html;
  238. $data['url'] = $result['url']; // this will be the effective URL after following redirects
  239. $data['code'] = $result['code'];
  240. return $this->respond($response, 200, $data);
  241. }
  242. }
  243. // TODO: look for other content like OEmbed or other known services later
  244. return $this->respond($response, 200, [
  245. 'data' => [
  246. 'type' => 'unknown',
  247. ],
  248. 'url' => $result['url'],
  249. 'code' => $result['code']
  250. ]);
  251. }
  252. private static function xPathFindNodeWithAttribute($xpath, $node, $attr, $callback) {
  253. foreach($xpath->query('//'.$node.'[@'.$attr.']') as $el) {
  254. $v = $el->getAttribute($attr);
  255. $callback($v);
  256. }
  257. }
  258. private static function xPathGetElementById($xpath, $id) {
  259. $element = null;
  260. foreach($xpath->query("//*[@id='$id']") as $el) {
  261. $element = $el;
  262. }
  263. return $element;
  264. }
  265. private function parseTwitterURL(&$request, &$response, $url, $match) {
  266. $fields = ['twitter_api_key','twitter_api_secret','twitter_access_token','twitter_access_token_secret'];
  267. $creds = [];
  268. foreach($fields as $f) {
  269. if($v=$request->get($f))
  270. $creds[$f] = $v;
  271. }
  272. $data = false;
  273. if(count($creds) == 4) {
  274. list($data, $parsed) = Formats\Twitter::parse($url, $match[1], $creds);
  275. } elseif(count($creds) > 0) {
  276. // If only some Twitter credentials were present, return an error
  277. return $this->respond($response, 400, [
  278. 'error' => 'missing_parameters',
  279. 'error_description' => 'All 4 Twitter credentials must be included in the request'
  280. ]);
  281. } else {
  282. // Accept Tweet JSON and parse that if provided
  283. $json = $request->get('json');
  284. if($json) {
  285. list($data, $parsed) = Formats\Twitter::parse($url, $match[1], null, $json);
  286. }
  287. // Skip parsing from the Twitter API if they didn't include credentials
  288. }
  289. if($data) {
  290. if($request->get('include_original'))
  291. $data['original'] = $parsed;
  292. $data['url'] = $url;
  293. $data['code'] = 200;
  294. return $this->respond($response, 200, $data);
  295. } else {
  296. return $this->respond($response, 200, [
  297. 'data' => [
  298. 'type' => 'unknown'
  299. ],
  300. 'url' => $url,
  301. 'code' => 0
  302. ]);
  303. }
  304. }
  305. }