You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

216 lines
6.1 KiB

3 years ago
  1. <?php
  2. namespace p3k\XRay;
  3. class Fetcher {
  4. private $http;
  5. public function __construct($http) {
  6. $this->http = $http;
  7. }
  8. public function fetch($url, $opts=[]) {
  9. if($opts == false) $opts = [];
  10. if(isset($opts['timeout']))
  11. $this->http->set_timeout($opts['timeout']);
  12. if(isset($opts['max_redirects']))
  13. $this->http->set_max_redirects($opts['max_redirects']);
  14. // Attempt some basic URL validation
  15. $scheme = parse_url($url, PHP_URL_SCHEME);
  16. if(!in_array($scheme, ['http','https'])) {
  17. return [
  18. 'error_code' => 400,
  19. 'error' => 'invalid_url',
  20. 'error_description' => 'Only http and https URLs are supported'
  21. ];
  22. }
  23. $host = parse_url($url, PHP_URL_HOST);
  24. if(!$host) {
  25. return [
  26. 'error_code' => 400,
  27. 'error' => 'invalid_url',
  28. 'error_description' => 'The URL provided was not valid'
  29. ];
  30. }
  31. $url = normalize_url($url);
  32. $host = parse_url($url, PHP_URL_HOST);
  33. // Check if this is a Twitter URL and use the API
  34. if(Formats\Twitter::matches_host($url)) {
  35. return $this->_fetch_tweet($url, $opts);
  36. }
  37. // Check if this is a Facebook URL and use the API
  38. if(Formats\Facebook::matches_host($url)) {
  39. return $this->_fetch_facebook($url, $opts);
  40. }
  41. // Transform the HTML GitHub URL into an GitHub API request and fetch the API response
  42. if(Formats\GitHub::matches_host($url)) {
  43. return $this->_fetch_github($url, $opts);
  44. }
  45. // Check if this is a Hackernews URL and use the API
  46. if(Formats\Hackernews::matches($url)) {
  47. return Formats\Hackernews::fetch($this->http, $url, $opts);
  48. }
  49. // All other URLs are fetched normally
  50. // Special-case appspot.com URLs to not follow redirects.
  51. // https://cloud.google.com/appengine/docs/php/urlfetch/
  52. if(!should_follow_redirects($url)) {
  53. $this->http->set_max_redirects(0);
  54. $this->http->set_transport(new \p3k\HTTP\Stream());
  55. } else {
  56. $this->http->set_transport(new \p3k\HTTP\Curl());
  57. }
  58. $headers = [];
  59. $headers[] = 'Accept: application/mf2+json, application/activity+json, text/html, application/json, application/xml, text/xml';
  60. if(isset($opts['token']))
  61. $headers[] = 'Authorization: Bearer ' . $opts['token'];
  62. $result = $this->http->get($url, $headers);
  63. if($result['error']) {
  64. return [
  65. 'error' => $result['error'],
  66. 'error_description' => $result['error_description'],
  67. 'url' => $result['url'],
  68. 'code' => $result['code'],
  69. ];
  70. }
  71. // Show an error if the content type returned is not a recognized type
  72. $format = null;
  73. if(isset($result['headers']['Content-Type'])) {
  74. $contentType = null;
  75. if(is_array($result['headers']['Content-Type'])) {
  76. $contentType = $result['headers']['Content-Type'][0];
  77. } elseif(is_string($result['headers']['Content-Type'])) {
  78. $contentType = $result['headers']['Content-Type'];
  79. }
  80. if($contentType) {
  81. $type = new MediaType($contentType);
  82. $format = $type->format;
  83. }
  84. }
  85. if(!$format ||
  86. !in_array($format, ['html', 'json', 'xml'])) {
  87. return [
  88. 'error' => 'invalid_content',
  89. 'error_description' => 'The server did not return a recognized content type',
  90. 'content_type' => isset($result['headers']['Content-Type']) ? $result['headers']['Content-Type'] : null,
  91. 'url' => $result['url'],
  92. 'code' => $result['code']
  93. ];
  94. }
  95. if(trim($result['body']) == '') {
  96. if($result['code'] == 410) {
  97. // 410 Gone responses are valid and should not return an error
  98. return [
  99. 'data' => [
  100. 'type' => 'deleted'
  101. ],
  102. 'url' => $result['url'],
  103. 'code' => $result['code']
  104. ];
  105. }
  106. return [
  107. 'error' => 'no_content',
  108. 'error_description' => 'We did not get a response body when fetching the URL',
  109. 'url' => $result['url'],
  110. 'code' => $result['code']
  111. ];
  112. }
  113. // Check for HTTP 401/403
  114. if($result['code'] == 401) {
  115. return [
  116. 'error' => 'unauthorized',
  117. 'error_description' => 'The URL returned "HTTP 401 Unauthorized"',
  118. 'url' => $result['url'],
  119. 'code' => $result['code']
  120. ];
  121. }
  122. if($result['code'] == 403) {
  123. return [
  124. 'error' => 'forbidden',
  125. 'error_description' => 'The URL returned "HTTP 403 Forbidden"',
  126. 'url' => $result['url'],
  127. 'code' => $result['code']
  128. ];
  129. }
  130. // If the original URL had a fragment, include it in the final URL
  131. if(($fragment=parse_url($url, PHP_URL_FRAGMENT)) && !parse_url($result['url'], PHP_URL_FRAGMENT)) {
  132. $result['url'] .= '#'.$fragment;
  133. }
  134. return [
  135. 'url' => $result['url'],
  136. 'body' => $result['body'],
  137. 'code' => $result['code'],
  138. ];
  139. }
  140. private function _fetch_tweet($url, $opts) {
  141. $fields = ['twitter_api_key','twitter_api_secret','twitter_access_token','twitter_access_token_secret'];
  142. $creds = [];
  143. foreach($fields as $f) {
  144. if(isset($opts[$f]))
  145. $creds[$f] = $opts[$f];
  146. }
  147. if(count($creds) < 4) {
  148. return [
  149. 'error_code' => 400,
  150. 'error' => 'missing_parameters',
  151. 'error_description' => 'All 4 Twitter credentials must be included in the request'
  152. ];
  153. }
  154. return Formats\Twitter::fetch($url, $creds);
  155. }
  156. private function _fetch_facebook($url, $opts) {
  157. $fields = ['facebook_app_id','facebook_app_secret'];
  158. $creds = [];
  159. foreach($fields as $f) {
  160. if(isset($opts[$f]))
  161. $creds[$f] = $opts[$f];
  162. }
  163. if(count($creds) < 2) {
  164. return [
  165. 'error_code' => 400,
  166. 'error' => 'missing_parameters',
  167. 'error_description' => 'Both Facebook credentials must be included in the request'
  168. ];
  169. }
  170. // TODO: Question, should I do this like Twitter or like Github?
  171. return Formats\Facebook::fetch($url, $creds);
  172. }
  173. private function _fetch_github($url, $opts) {
  174. $fields = ['github_access_token'];
  175. $creds = [];
  176. foreach($fields as $f) {
  177. if(isset($opts[$f]))
  178. $creds[$f] = $opts[$f];
  179. }
  180. return Formats\GitHub::fetch($this->http, $url, $creds);
  181. }
  182. }