You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

242 lines
7.9 KiB

  1. <?php
  2. namespace p3k\XRay;
  3. use p3k\XRay\Formats;
  4. use DOMDocument, DOMXPath;
  5. class Parser {
  6. private $http;
  7. public function __construct($http) {
  8. $this->http = $http;
  9. }
  10. public function parse($http_response, $opts=[]) {
  11. $document = $this->parse_document($http_response, $opts);
  12. // If a target parameter was provided, make sure a link to it exists in the parsed document
  13. if(!isset($document['error']) && !empty($opts['target'])) {
  14. if(isset($document['data']['type']) && $document['data']['type'] == 'unknown') {
  15. if(isset($document['html'])) {
  16. // Couldn't parse the page, check for the link manually assuming HTML content
  17. $found = $this->_findLinkInHTML($opts['target'], $document['html']);
  18. } else {
  19. // Ignore this check for any non-HTML documents since this will be uncommon anyway
  20. $found = false;
  21. }
  22. $error_description = 'The source document does not have a link to the target URL';
  23. } else {
  24. $found = $this->_findLinkInTree($opts['target'], $document['data']);
  25. $error_description = 'The Microformats at the source URL do not contain a link to the target URL. Check the source URL in a Microformats parser such as php.microformats.io';
  26. if(!$found && isset($document['html'])) {
  27. // If no link was found in the parsed mf2 tree, check for a link in the HTML
  28. $found = $this->_findLinkInHTML($opts['target'], $document['html']);
  29. // If there is a link, and if the HTML document has no mf2, then downgrade to a regular mention
  30. if($found) {
  31. $mf2Data = Formats\HTML::parse($this->http, $http_response, ['include-mf1'=>false]);
  32. if(isset($mf2Data['data']['type']) && $mf2Data['data']['type'] == 'unknown') {
  33. // Since the link was found in the HTML, but not in the parsed tree, it shouldn't return the parsed document
  34. $document['data'] = [
  35. 'type' => 'unknown'
  36. ];
  37. } else {
  38. // Otherwise, the document did have mf2, but the link wasn't in it (checked earlier), so set found=false
  39. $found = false;
  40. }
  41. }
  42. }
  43. }
  44. if(!$found) {
  45. return [
  46. 'error' => 'no_link_found',
  47. 'error_description' => $error_description,
  48. 'code' => isset($document['code']) ? $document['code'] : 200,
  49. 'url' => $document['url'],
  50. 'debug' => $document['data']
  51. ];
  52. }
  53. }
  54. // If the HTML parser couldn't parse the page it returns the full HTML for checking the target above,
  55. // but we don't want to return that in the output so remove it here
  56. unset($document['html']);
  57. return $document;
  58. }
  59. public function parse_document($http_response, $opts=[]) {
  60. if(isset($opts['timeout']))
  61. $this->http->set_timeout($opts['timeout']);
  62. if(isset($opts['max_redirects']))
  63. $this->http->set_max_redirects($opts['max_redirects']);
  64. // Check if the URL matches a special parser
  65. $url = $http_response['url'];
  66. if(Formats\Instagram::matches($url)) {
  67. return Formats\Instagram::parse($this->http, $http_response, $opts);
  68. }
  69. if(Formats\GitHub::matches($url)) {
  70. return Formats\GitHub::parse($http_response);
  71. }
  72. if(Formats\Twitter::matches($url)) {
  73. return Formats\Twitter::parse($http_response);
  74. }
  75. if(Formats\Facebook::matches($url)) {
  76. return Formats\Facebook::parse($http_response);
  77. }
  78. if(Formats\XKCD::matches($url)) {
  79. return Formats\XKCD::parse($http_response);
  80. }
  81. if(Formats\Hackernews::matches($url)) {
  82. return Formats\Hackernews::parse($http_response);
  83. }
  84. $body = $http_response['body'];
  85. // Check if an mf2 JSON object was passed in
  86. if(is_array($body) && isset($body['items'])) {
  87. $data = Formats\Mf2::parse($http_response, $this->http, $opts);
  88. if($data == false) {
  89. $data = [
  90. 'data' => [
  91. 'type' => 'unknown',
  92. ]
  93. ];
  94. }
  95. $data['source-format'] = 'mf2+json';
  96. return $data;
  97. }
  98. // Check if an ActivityStreams JSON object was passed in
  99. if(Formats\ActivityStreams::is_as2_json($body)) {
  100. $data = Formats\ActivityStreams::parse($http_response, $this->http, $opts);
  101. $data['source-format'] = 'activity+json';
  102. return $data;
  103. }
  104. if(is_string($body) && substr($body, 0, 5) == '<?xml') {
  105. return Formats\XML::parse($http_response);
  106. }
  107. if(is_string($body)) {
  108. // Some feeds don't start with <?xml
  109. $begin = trim(substr($body, 0, 40));
  110. if(substr($begin, 0, 4) == '<rss') {
  111. return Formats\XML::parse($http_response);
  112. }
  113. }
  114. if(is_string($body) && substr($body, 0, 1) == '{') {
  115. $parsed = json_decode($body, true);
  116. if($parsed && isset($parsed['version']) && $parsed['version'] == 'https://jsonfeed.org/version/1') {
  117. $http_response['body'] = $parsed;
  118. return Formats\JSONFeed::parse($http_response);
  119. } elseif($parsed && isset($parsed['items'][0]['type']) && isset($parsed['items'][0]['properties'])) {
  120. // Check if an mf2 JSON string was passed in
  121. $http_response['body'] = $parsed;
  122. $data = Formats\Mf2::parse($http_response, $this->http, $opts);
  123. $data['source-format'] = 'mf2+json';
  124. return $data;
  125. } elseif($parsed && Formats\ActivityStreams::is_as2_json($parsed)) {
  126. // Check if an ActivityStreams JSON string was passed in
  127. $http_response['body'] = $parsed;
  128. $data = Formats\ActivityStreams::parse($http_response, $this->http, $opts);
  129. $data['source-format'] = 'activity+json';
  130. return $data;
  131. }
  132. }
  133. // No special parsers matched, parse for Microformats now
  134. $data = Formats\HTML::parse($this->http, $http_response, $opts);
  135. if(!isset($data['source-format']) && isset($data['type']) && $data['type'] != 'unknown')
  136. $data['source-format'] = 'mf2+html';
  137. return $data;
  138. }
  139. private function _findLinkInTree($link, $document) {
  140. if(!$document)
  141. return false;
  142. if(is_string($document) || is_numeric($document)) {
  143. return $document == $link;
  144. }
  145. if(is_array($document)) {
  146. foreach($document as $key=>$value) {
  147. if($key === 'html') {
  148. $found = $this->_findLinkInHTML($link, $value);
  149. if($found) {
  150. return true;
  151. }
  152. } else {
  153. $found = $this->_findLinkInTree($link, $value);
  154. if($found) {
  155. return true;
  156. }
  157. }
  158. }
  159. return false;
  160. }
  161. throw new Exception('Unexpected value in tree');
  162. }
  163. private function _findLinkInHTML($link, $html) {
  164. $doc = new DOMDocument();
  165. @$doc->loadHTML(self::_toHtmlEntities($html));
  166. if(!$doc)
  167. return false;
  168. $xpath = new DOMXPath($doc);
  169. return self::_findLinksInDOMDocument($xpath, $link);
  170. }
  171. private static function _findLinksInDOMDocument(&$xpath, $target) {
  172. $found = [];
  173. self::_xPathFindNodeWithAttribute($xpath, 'a', 'href', function($u) use($target, &$found){
  174. if($u == $target) {
  175. $found[$u] = null;
  176. }
  177. });
  178. self::_xPathFindNodeWithAttribute($xpath, 'img', 'src', function($u) use($target, &$found){
  179. if($u == $target) {
  180. $found[$u] = null;
  181. }
  182. });
  183. self::_xPathFindNodeWithAttribute($xpath, 'video', 'src', function($u) use($target, &$found){
  184. if($u == $target) {
  185. $found[$u] = null;
  186. }
  187. });
  188. self::_xPathFindNodeWithAttribute($xpath, 'audio', 'src', function($u) use($target, &$found){
  189. if($u == $target) {
  190. $found[$u] = null;
  191. }
  192. });
  193. return $found;
  194. }
  195. private static function _xPathFindNodeWithAttribute($xpath, $node, $attr, $callback) {
  196. foreach($xpath->query('//'.$node.'[@'.$attr.']') as $el) {
  197. $v = $el->getAttribute($attr);
  198. $callback($v);
  199. }
  200. }
  201. private static function _toHtmlEntities($input) {
  202. return mb_convert_encoding($input, 'HTML-ENTITIES', mb_detect_encoding($input));
  203. }
  204. }