You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

159 lines
4.9 KiB

  1. <?php
  2. namespace p3k\XRay\Formats;
  3. use HTMLPurifier, HTMLPurifier_Config;
  4. use DOMDocument, DOMXPath;
  5. use p3k\XRay\Formats;
  6. class HTML extends Format {
  7. public static function matches_host($url) { return true; }
  8. public static function matches($url) { return true; }
  9. public static function parse($http, $http_response, $opts=[]) {
  10. $html = $http_response['body'];
  11. $url = $http_response['url'];
  12. $result = [
  13. 'data' => [
  14. 'type' => 'unknown',
  15. ],
  16. 'url' => $url,
  17. 'code' => $http_response['code'],
  18. 'html' => $html,
  19. ];
  20. // attempt to parse the page as HTML
  21. $doc = new DOMDocument();
  22. @$doc->loadHTML(self::toHtmlEntities($html));
  23. if(!$doc) {
  24. return [
  25. 'error' => 'invalid_content',
  26. 'error_description' => 'The document could not be parsed as HTML'
  27. ];
  28. }
  29. $xpath = new DOMXPath($doc);
  30. // Check for meta http equiv and replace the status code if present
  31. foreach($xpath->query('//meta[translate(@http-equiv,\'STATUS\',\'status\')=\'status\']') as $el) {
  32. $equivStatus = ''.$el->getAttribute('content');
  33. if($equivStatus && is_string($equivStatus)) {
  34. if(preg_match('/^(\d+)/', $equivStatus, $match)) {
  35. $result['code'] = (int)$match[1];
  36. }
  37. }
  38. }
  39. // If the URL has a fragment ID, find the DOM starting at that node and parse it instead
  40. $fragment = parse_url($url, PHP_URL_FRAGMENT);
  41. if($fragment) {
  42. $fragElement = self::xPathGetElementById($xpath, $fragment);
  43. if($fragElement) {
  44. $html = $doc->saveHTML($fragElement);
  45. $foundFragment = true;
  46. } else {
  47. $foundFragment = false;
  48. }
  49. }
  50. $mf2 = \mf2\Parse($html, $url);
  51. // Check for a rel=alternate link to a Microformats JSON representation, and use that instead
  52. if(isset($mf2['rel-urls'])) {
  53. $alternates = [
  54. 'mf2' => [],
  55. 'as2' => [],
  56. ];
  57. foreach($mf2['rel-urls'] as $relurl => $reltype) {
  58. if(isset($reltype['type'])) {
  59. if(in_array('alternate', $reltype['rels']) && $reltype['type'] == 'application/mf2+json') {
  60. $alternates['mf2'][] = $relurl;
  61. }
  62. if(in_array('alternate', $reltype['rels']) && $reltype['type'] == 'application/activity+json') {
  63. $alternates['as2'][] = $relurl;
  64. }
  65. }
  66. }
  67. if(count($alternates['mf2'])) {
  68. // Fetch and parse the MF2 JSON link
  69. $relurl = $alternates['mf2'][0];
  70. $jsonpage = $http->get($relurl, [
  71. 'Accept' => 'application/mf2+json,application/json'
  72. ]);
  73. // Skip and fall back to parsing the HTML if anything about this request fails
  74. if(!$jsonpage['error'] && $jsonpage['body']) {
  75. $jsondata = json_decode($jsonpage['body'], true);
  76. if($jsondata) {
  77. $jsonpage['body'] = $jsondata;
  78. $data = Formats\Mf2::parse($jsonpage, $http, $opts);
  79. if($data && is_array($data) && isset($data['data']['type'])) {
  80. $data['url'] = $relurl;
  81. $data['source-format'] = 'mf2+json';
  82. return $data;
  83. }
  84. }
  85. }
  86. }
  87. if(count($alternates['as2'])) {
  88. $relurl = $alternates['as2'][0];
  89. // Fetch and parse the ActivityStreams JSON link
  90. $jsonpage = $http->get($relurl, [
  91. 'Accept' => 'application/activity+json,application/json'
  92. ]);
  93. // Skip and fall back to parsing the HTML if anything about this request fails
  94. if(!$jsonpage['error'] && $jsonpage['body']) {
  95. $jsondata = json_decode($jsonpage['body'],true);
  96. if($jsondata) {
  97. $jsonpage['body'] = $jsondata;
  98. $data = Formats\ActivityStreams::parse($jsonpage, $http, $opts);
  99. if($data && is_array($data) && isset($data['data']['type'])) {
  100. $data['url'] = $relurl;
  101. $data['source-format'] = 'activity+json';
  102. return $data;
  103. }
  104. }
  105. }
  106. }
  107. }
  108. // Now start pulling in the data from the page. Start by looking for microformats2
  109. if($mf2 && count($mf2['items']) > 0) {
  110. $http_response['body'] = $mf2;
  111. $data = Formats\Mf2::parse($http_response, $http, $opts);
  112. if($data) {
  113. $result = array_merge($result, $data);
  114. if($fragment) {
  115. $result['info'] = [
  116. 'found_fragment' => $foundFragment
  117. ];
  118. }
  119. $result['original'] = $html;
  120. $result['url'] = $url; // this will be the effective URL after following redirects
  121. $result['source-format'] = 'mf2+html';
  122. }
  123. }
  124. return $result;
  125. }
  126. private static function toHtmlEntities($input) {
  127. return mb_convert_encoding($input, 'HTML-ENTITIES', mb_detect_encoding($input));
  128. }
  129. private static function xPathGetElementById($xpath, $id) {
  130. $element = null;
  131. foreach($xpath->query("//*[@id='$id']") as $el) {
  132. $element = $el;
  133. }
  134. return $element;
  135. }
  136. }