add twitter support

closes #18

README.md

@@ -8,6 +8,7 @@ The contents of the URL is checked in the following order:
 
 * A silo URL from one of the following websites:
 ** Instagram
+** Twitter
 ** (more coming soon)
 * h-entry, h-event, h-card
 * OEmbed (coming soon)
@@ -44,6 +45,18 @@ url=https://aaronparecki.com/2016/01/16/11/
 &token=12341234123412341234
 ```
 
+### Twitter Authentication
+
+XRay uses the Twitter API to fetch posts, and the Twitter API requires authentication. In order to keep XRay stateless, it is required that you pass in Twitter credentials to the parse call. You can register an application on the Twitter developer website, and generate an access token for your account without writing any code, and then use those credentials when making an API request to XRay.
+
+You should only send Twitter credentials when the URL you are trying to parse is a Twitter URL, so you'll want to check for whether the hostname is `twitter.com` before you include credentials in this call.
+
+* twitter_api_key - Your application's API key
+* twitter_api_secret - Your application's API secret
+* twitter_access_token - Your Twitter access token
+* twitter_access_token_secret - Your Twitter secret access token
+
+
 ### Error Response
 
 ```json

composer.json

@@ -4,7 +4,8 @@
     "league/route": "1.*",
     "mf2/mf2": "~0.3",
     "ezyang/htmlpurifier": "4.*",
-    "indieweb/link-rel-parser": "0.1.*"
+    "indieweb/link-rel-parser": "0.1.*",
+    "dg/twitter-php": "^3.6"
   },
   "autoload": {
     "files": [
@@ -19,6 +20,7 @@
       "lib/HTTP.php",
       "lib/Formats/Mf2.php",
       "lib/Formats/Instagram.php",
+      "lib/Formats/Twitter.php",
       "lib/Formats/HTMLPurifier_AttrDef_HTML_Microformats2.php"
     ]
   },

composer.lock

@@ -4,9 +4,50 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "hash": "b55b9f1fabddb79bf5e8b0daf6b4f88e",
-    "content-hash": "a791e61b2f956830a8b7d9b0a2493148",
+    "content-hash": "e5c15a7976be20fd9ed6f7cc48096672",
     "packages": [
+        {
+            "name": "dg/twitter-php",
+            "version": "v3.6",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/dg/twitter-php.git",
+                "reference": "dd872ad12121ff919b358989e61f7f08ba6cc7a8"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/dg/twitter-php/zipball/dd872ad12121ff919b358989e61f7f08ba6cc7a8",
+                "reference": "dd872ad12121ff919b358989e61f7f08ba6cc7a8",
+                "shasum": ""
+            },
+            "require": {
+                "ext-curl": "*",
+                "php": ">=5.2.0"
+            },
+            "type": "library",
+            "autoload": {
+                "classmap": [
+                    "src/"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "BSD-3-Clause"
+            ],
+            "authors": [
+                {
+                    "name": "David Grudl",
+                    "homepage": "https://davidgrudl.com"
+                }
+            ],
+            "description": "Small and easy Twitter library for PHP",
+            "homepage": "https://github.com/dg/twitter-php",
+            "keywords": [
+                "oauth",
+                "twitter"
+            ],
+            "time": "2016-08-15T16:46:22+00:00"
+        },
         {
             "name": "ezyang/htmlpurifier",
             "version": "v4.8.0",
@@ -49,7 +90,7 @@
             "keywords": [
                 "html"
             ],
-            "time": "2016-07-16 12:58:58"
+            "time": "2016-07-16T12:58:58+00:00"
         },
         {
             "name": "indieweb/link-rel-parser",
@@ -95,7 +136,7 @@
                 "indieweb",
                 "microformats2"
             ],
-            "time": "2016-04-13 17:48:59"
+            "time": "2016-04-13T17:48:59+00:00"
         },
         {
             "name": "ircmaxell/password-compat",
@@ -137,7 +178,7 @@
                 "hashing",
                 "password"
             ],
-            "time": "2014-11-20 16:49:30"
+            "time": "2014-11-20T16:49:30+00:00"
         },
         {
             "name": "league/container",
@@ -195,7 +236,7 @@
                 "injection",
                 "league"
             ],
-            "time": "2015-04-05 17:14:48"
+            "time": "2015-04-05T17:14:48+00:00"
         },
         {
             "name": "league/plates",
@@ -247,7 +288,7 @@
                 "templating",
                 "views"
             ],
-            "time": "2015-07-09 02:14:40"
+            "time": "2015-07-09T02:14:40+00:00"
         },
         {
             "name": "league/route",
@@ -305,7 +346,7 @@
                 "league",
                 "route"
             ],
-            "time": "2015-09-11 07:40:31"
+            "time": "2015-09-11T07:40:31+00:00"
         },
         {
             "name": "mf2/mf2",
@@ -358,7 +399,7 @@
                 "parser",
                 "semantic"
             ],
-            "time": "2016-03-14 12:13:34"
+            "time": "2016-03-14T12:13:34+00:00"
         },
         {
             "name": "nikic/fast-route",
@@ -401,7 +442,7 @@
                 "router",
                 "routing"
             ],
-            "time": "2016-03-25 23:46:52"
+            "time": "2016-03-25T23:46:52+00:00"
         },
         {
             "name": "symfony/http-foundation",
@@ -456,7 +497,7 @@
             ],
             "description": "Symfony HttpFoundation Component",
             "homepage": "https://symfony.com",
-            "time": "2016-03-27 12:57:53"
+            "time": "2016-03-27T12:57:53+00:00"
         },
         {
             "name": "symfony/polyfill-mbstring",
@@ -515,7 +556,7 @@
                 "portable",
                 "shim"
             ],
-            "time": "2016-01-20 09:13:37"
+            "time": "2016-01-20T09:13:37+00:00"
         },
         {
             "name": "symfony/polyfill-php54",
@@ -573,7 +614,7 @@
                 "portable",
                 "shim"
             ],
-            "time": "2016-01-25 19:13:00"
+            "time": "2016-01-25T19:13:00+00:00"
         },
         {
             "name": "symfony/polyfill-php55",
@@ -629,7 +670,7 @@
                 "portable",
                 "shim"
             ],
-            "time": "2016-01-20 09:13:37"
+            "time": "2016-01-20T09:13:37+00:00"
         }
     ],
     "packages-dev": [],

controllers/Parse.php

@@ -90,6 +90,45 @@ class Parse {
 
       $url = \normalize_url($url);
 
+      // Check if this is a Twitter URL and if they've provided API credentials, use the API
+      if(preg_match('/https?:\/\/(?:mobile\.twitter\.com|twitter\.com|twtr\.io)\/(?:[a-z0-9_\/!#]+statuse?s?\/([0-9]+)|([a-zA-Z0-9_]+))/', $url, $match)) {
+        $fields = ['twitter_api_key','twitter_api_secret','twitter_access_token','twitter_access_token_secret'];
+        $creds = [];
+        foreach($fields as $f) {
+          if($v=$request->get($f))
+            $creds[$f] = $v;
+        }
+        $data = false;
+        if(count($creds) == 4) {
+          list($data, $parsed) = Formats\Twitter::parse($url, $match[1], $creds);
+        } elseif(count($creds) > 0) {
+          // If only some Twitter credentials were present, return an error  
+          return $this->respond($response, 400, [
+            'error' => 'missing_parameters',
+            'error_description' => 'All 4 Twitter credentials must be included in the request'
+          ]);
+        } else {
+          // Accept Tweet JSON and parse that if provided
+          $json = $request->get('json');
+          if($json) {
+            list($data, $parsed) = Formats\Twitter::parse($url, $match[1], null, $json);
+          }
+          // Skip parsing from the Twitter API if they didn't include credentials
+        }
+
+        if($data) {
+          if($request->get('include_original'))
+            $data['original'] = $parsed;
+          return $this->respond($response, 200, $data);
+        } else {
+          return $this->respond($response, 200, [
+            'data' => [
+              'type' => 'unknown'
+            ]
+          ]);
+        }
+      }
+
       // Now fetch the URL and check for any curl errors
       // Don't cache the response if a token is used to fetch it
       if($this->mc && !$request->get('token')) {
@@ -145,6 +184,7 @@ class Parse {
 
     // Check for known services
     $host = parse_url($result['url'], PHP_URL_HOST);
+
     if(in_array($host, ['www.instagram.com','instagram.com'])) {
       list($data, $parsed) = Formats\Instagram::parse($result['body'], $result['url'], $this->http);
       if($request->get('include_original'))
@@ -152,7 +192,6 @@ class Parse {
       return $this->respond($response, 200, $data);
     }
 
-
     // attempt to parse the page as HTML
     $doc = new DOMDocument();
     @$doc->loadHTML(self::toHtmlEntities($result['body']));

lib/Formats/Twitter.php

@@ -0,0 +1,231 @@
+<?php
+namespace XRay\Formats;
+
+use DOMDocument, DOMXPath;
+use DateTime, DateTimeZone;
+use Parse;
+
+class Twitter {
+
+  public static function parse($url, $tweet_id, $creds, $json=null) {
+
+    $host = parse_url($url, PHP_URL_HOST);
+    if($host == 'twtr.io') {
+      $tweet_id = self::b60to10($tweet_id);
+    }
+
+    if($json) {
+      if(is_string($json))
+        $tweet = json_decode($json);
+      else
+        $tweet = $json;
+    } else {
+      $twitter = new \Twitter($creds['twitter_api_key'], $creds['twitter_api_secret'], $creds['twitter_access_token'], $creds['twitter_access_token_secret']);
+      $tweet = $twitter->request('statuses/show/'.$tweet_id, 'GET', ['tweet_mode'=>'extended']);
+    }
+
+    if(!$tweet)
+      return false;
+
+    $entry = array(
+      'type' => 'entry',
+      'url' => $url,
+      'author' => [
+        'type' => 'card',
+        'name' => null,
+        'nickname' => null,
+        'photo' => null,
+        'url' => null
+      ]
+    );
+    $refs = [];
+
+    // Only use the "display" segment of the text
+    $text = mb_substr($tweet->full_text, $tweet->display_text_range[0], $tweet->display_text_range[1]-$tweet->display_text_range[0]);
+
+    if(property_exists($tweet, 'retweeted_status')) {
+      // No content for retweets
+
+      $reposted = $tweet->retweeted_status;
+      $repostOf = 'https://twitter.com/' . $reposted->user->screen_name . '/status/' . $reposted->id_str;
+      $entry['repost-of'] = $repostOf;
+
+      list($repostedEntry) = self::parse($repostOf, $reposted->id_str, null, $reposted);
+      if(isset($repostedEntry['refs'])) {
+        foreach($repostedEntry['refs'] as $k=>$v) {
+          $refs[$k] = $v;
+        }
+      }
+
+      $refs[$repostOf] = $repostedEntry['data'];
+
+    } else {
+      // Twitter escapes & as &amp; in the text
+      $text = html_entity_decode($text);
+
+      $text = self::expandTweetURLs($text, $tweet);
+
+      $entry['content'] = ['text' => $text];
+    }
+
+    // Published date
+    $published = new DateTime($tweet->created_at);
+    if(property_exists($tweet->user, 'utc_offset')) {
+      $tz = new DateTimeZone($tweet->user->utc_offset / 3600);
+      $published->setTimeZone($tz);
+    }
+    $entry['published'] = $published->format('c');
+
+    // Hashtags
+    if(property_exists($tweet, 'entities') && property_exists($tweet->entities, 'hashtags')) {
+      if(count($tweet->entities->hashtags)) {
+        $entry['category'] = [];
+        foreach($tweet->entities->hashtags as $hashtag) {
+          $entry['category'][] = $hashtag->text;
+        }
+      }
+    }
+
+    // Photos and Videos
+    if(property_exists($tweet, 'extended_entities') && property_exists($tweet->extended_entities, 'media')) {
+      foreach($tweet->extended_entities->media as $media) {
+        if($media->type == 'photo') {
+          if(!array_key_exists('photo', $entry))
+            $entry['photo'] = [];
+
+          $entry['photo'][] = $media->media_url_https;
+
+        } elseif($media->type == 'video') {
+          if(!array_key_exists('video', $entry))
+            $entry['video'] = [];
+
+          // Find the highest bitrate video that is mp4
+          $videos = $media->video_info->variants;
+          $videos = array_filter($videos, function($v) {
+            return property_exists($v, 'bitrate') && $v->content_type == 'video/mp4';
+          });
+          if(count($videos)) {
+            usort($videos, function($a,$b) {
+              return $a->bitrate < $b->bitrate;
+            });
+            $entry['video'][] = $videos[0]->url;
+          }
+        }
+      }
+    }
+
+    // Place
+    if(property_exists($tweet, 'place') && $tweet->place) {
+      $place = $tweet->place;
+      if($place->place_type == 'city') {
+        $entry['location'] = $place->url;
+        $refs[$place->url] = [
+          'type' => 'adr',
+          'name' => $place->full_name,
+          'locality' => $place->name,
+          'country-name' => $place->country,
+        ];
+      }
+    }
+
+    // Quoted Status
+    if(property_exists($tweet, 'quoted_status')) {
+      $quoteOf = 'https://twitter.com/' . $tweet->quoted_status->user->screen_name . '/status/' . $tweet->quoted_status_id_str;
+      list($quoted) = self::parse($quoteOf, $tweet->quoted_status_id_str, null, $tweet->quoted_status);
+      if(isset($quoted['refs'])) {
+        foreach($quoted['refs'] as $k=>$v) {
+          $refs[$k] = $v;
+        }
+      }
+      $refs[$quoteOf] = $quoted['data'];
+    }
+
+    if($author = self::_buildHCardFromTwitterProfile($tweet->user)) {
+      $entry['author'] = $author;
+    }
+
+    $response = [
+      'data' => $entry
+    ];
+
+    if(count($refs)) {
+      $response['refs'] = $refs;
+    }
+
+    return [$response, $tweet];
+  }
+
+  private static function _buildHCardFromTwitterProfile($profile) {
+    if(!$profile) return false;
+
+    $author = [
+      'type' => 'card'
+    ];
+
+    $author['nickname'] = $profile->screen_name;
+    $author['location'] = $profile->location;
+    $author['bio'] = self::expandTwitterObjectURLs($profile->description, $profile, 'description');
+
+    if($profile->name)
+      $author['name'] = $profile->name;
+    else
+      $author['name'] = $profile->screen_name;
+
+    if($profile->url)
+      $author['url'] = $profile->entities->url->urls[0]->expanded_url;
+    else
+      $author['url'] = 'https://twitter.com/' . $profile->screen_name;
+
+    $author['photo'] = $profile->profile_image_url_https;
+
+    return $author;
+  }
+
+  private static function expandTweetURLs($text, $object) {
+    if(property_exists($object, 'entities') && property_exists($object->entities, 'urls')) {
+      foreach($object->entities->urls as $url) {
+        $text = str_replace($url->url, $url->expanded_url, $text);
+      }
+    }
+    return $text;
+  }
+
+  private static function expandTwitterObjectURLs($text, $object, $key) {
+    if(property_exists($object, 'entities') 
+      && property_exists($object->entities, $key) 
+      && property_exists($object->entities->{$key}, 'urls')) {
+      foreach($object->entities->{$key}->urls as $url) {
+        $text = str_replace($url->url, $url->expanded_url, $text);
+      }
+    }
+    return $text;
+  }
+
+  /**
+   * Converts base 60 to base 10, with error checking
+   * http://tantek.pbworks.com/NewBase60
+   * @param string $s
+   * @return int
+   */
+  function b60to10($s)
+  {
+    $n = 0;
+    for($i = 0; $i < strlen($s); $i++) // iterate from first to last char of $s
+    {
+      $c = ord($s[$i]); //  put current ASCII of char into $c  
+      if ($c>=48 && $c<=57) { $c=bcsub($c,48); }
+      else if ($c>=65 && $c<=72) { $c=bcsub($c,55); }
+      else if ($c==73 || $c==108) { $c=1; } // typo capital I, lowercase l to 1
+      else if ($c>=74 && $c<=78) { $c=bcsub($c,56); }
+      else if ($c==79) { $c=0; } // error correct typo capital O to 0
+      else if ($c>=80 && $c<=90) { $c=bcsub($c,57); }
+      else if ($c==95) { $c=34; } // underscore
+      else if ($c>=97 && $c<=107) { $c=bcsub($c,62); }
+      else if ($c>=109 && $c<=122) { $c=bcsub($c,63); }
+      else { $c = 0; } // treat all other noise as 0
+      $n = bcadd(bcmul(60, $n), $c);
+    }
+    return $n;
+  }
+
+}

tests/InstagramTest.php

@@ -0,0 +1,94 @@
+<?php
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+
+class InstagramTest extends PHPUnit_Framework_TestCase {
+
+  private $http;
+
+  public function setUp() {
+    $this->client = new Parse();
+    $this->client->http = new p3k\HTTPTest(dirname(__FILE__).'/data/');
+    $this->client->mc = null;
+  }
+
+  private function parse($params) {
+    $request = new Request($params);
+    $response = new Response();
+    return $this->client->parse($request, $response);
+  }
+
+  public function testInstagramPhoto() {
+    $url = 'http://www.instagram.com/photo.html';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('2017-01-05T23:31:32+00:00', $data['data']['published']);
+    $this->assertContains('planning', $data['data']['category']);
+    $this->assertContains('2017', $data['data']['category']);
+    $this->assertEquals('Kind of crazy to see the whole year laid out like this. #planning #2017', $data['data']['content']['text']);
+    $this->assertEquals(1, count($data['data']['photo']));
+    $this->assertEquals(['https://scontent.cdninstagram.com/t51.2885-15/e35/15803256_1832278043695907_4846092951052353536_n.jpg?ig_cache_key=MTQyMTM1Nzk0NTMwNTEwMDkwNg%3D%3D.2'], $data['data']['photo']);
+    $this->assertEquals('http://aaronparecki.com/', $data['data']['author']['url']);
+    $this->assertEquals('Aaron Parecki', $data['data']['author']['name']);
+    $this->assertEquals('https://scontent.cdninstagram.com/t51.2885-19/s320x320/14240576_268350536897085_1129715662_a.jpg', $data['data']['author']['photo']);
+  }
+
+  public function testInstagramVideo() {
+    $url = 'http://www.instagram.com/video.html';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertContains('100daysofmusic', $data['data']['category']);
+    $this->assertEquals('Day 18. Maple and Spruce #100daysofmusic #100daysproject #the100dayproject https://aaronparecki.com/2017/01/07/14/day18', $data['data']['content']['text']);
+    $this->assertEquals(1, count($data['data']['photo']));
+    $this->assertEquals(['https://scontent.cdninstagram.com/t51.2885-15/s640x640/e15/15624670_548881701986735_8264383763249627136_n.jpg?ig_cache_key=MTQyMjkzMTczMTg0MjE3NjE3Nw%3D%3D.2'], $data['data']['photo']);
+    $this->assertEquals(1, count($data['data']['video']));
+    $this->assertEquals(['https://scontent.cdninstagram.com/t50.2886-16/15921147_1074837002642259_2269307616507199488_n.mp4'], $data['data']['video']);
+    $this->assertEquals('http://aaronparecki.com/', $data['data']['author']['url']);
+    $this->assertEquals('Aaron Parecki', $data['data']['author']['name']);
+    $this->assertEquals('https://scontent.cdninstagram.com/t51.2885-19/s320x320/14240576_268350536897085_1129715662_a.jpg', $data['data']['author']['photo']);
+  }
+
+  public function testInstagramPhotoWithPersonTag() {
+    $url = 'http://www.instagram.com/photo_with_person_tag.html';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals(2, count($data['data']['category']));
+    $this->assertContains('https://kmikeym.com/', $data['data']['category']);
+    $this->assertArrayHasKey('https://kmikeym.com/', $data['refs']);
+    $this->assertEquals(['type'=>'card','name'=>'Mike Merrill','url'=>'https://kmikeym.com/','photo'=>'https://scontent.cdninstagram.com/t51.2885-19/s320x320/12627953_686238411518831_1544976311_a.jpg'], $data['refs']['https://kmikeym.com/']);
+  }
+
+  public function testInstagramPhotoWithVenue() {
+    $url = 'http://www.instagram.com/photo_with_venue.html';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals(1, count($data['data']['location']));
+    $this->assertContains('https://www.instagram.com/explore/locations/109284789535230/', $data['data']['location']);
+    $this->assertArrayHasKey('https://www.instagram.com/explore/locations/109284789535230/', $data['refs']);
+    $venue = $data['refs']['https://www.instagram.com/explore/locations/109284789535230/'];
+    $this->assertEquals('XOXO Outpost', $venue['name']);
+    $this->assertEquals('45.5261002', $venue['latitude']);
+    $this->assertEquals('-122.6558081', $venue['longitude']);
+    // Setting a venue should set the timezone
+    $this->assertEquals('2016-12-10T21:48:56-08:00', $data['data']['published']);
+  }
+
+}

tests/ParseTest.php

@@ -383,77 +383,4 @@ class ParseTest extends PHPUnit_Framework_TestCase {
     $this->assertFalse($data['info']['found_fragment']);
   }
 
-  public function testInstagramPhoto() {
-    $url = 'http://www.instagram.com/photo.html';
-    $response = $this->parse(['url' => $url]);
-
-    $body = $response->getContent();
-    $this->assertEquals(200, $response->getStatusCode());
-    $data = json_decode($body, true);
-
-    $this->assertEquals('entry', $data['data']['type']);
-    $this->assertEquals('2017-01-05T23:31:32+00:00', $data['data']['published']);
-    $this->assertContains('planning', $data['data']['category']);
-    $this->assertContains('2017', $data['data']['category']);
-    $this->assertEquals('Kind of crazy to see the whole year laid out like this. #planning #2017', $data['data']['content']['text']);
-    $this->assertEquals(1, count($data['data']['photo']));
-    $this->assertEquals(['https://scontent.cdninstagram.com/t51.2885-15/e35/15803256_1832278043695907_4846092951052353536_n.jpg?ig_cache_key=MTQyMTM1Nzk0NTMwNTEwMDkwNg%3D%3D.2'], $data['data']['photo']);
-    $this->assertEquals('http://aaronparecki.com/', $data['data']['author']['url']);
-    $this->assertEquals('Aaron Parecki', $data['data']['author']['name']);
-    $this->assertEquals('https://scontent.cdninstagram.com/t51.2885-19/s320x320/14240576_268350536897085_1129715662_a.jpg', $data['data']['author']['photo']);
-  }
-
-  public function testInstagramVideo() {
-    $url = 'http://www.instagram.com/video.html';
-    $response = $this->parse(['url' => $url]);
-
-    $body = $response->getContent();
-    $this->assertEquals(200, $response->getStatusCode());
-    $data = json_decode($body, true);
-
-    $this->assertEquals('entry', $data['data']['type']);
-    $this->assertContains('100daysofmusic', $data['data']['category']);
-    $this->assertEquals('Day 18. Maple and Spruce #100daysofmusic #100daysproject #the100dayproject https://aaronparecki.com/2017/01/07/14/day18', $data['data']['content']['text']);
-    $this->assertEquals(1, count($data['data']['photo']));
-    $this->assertEquals(['https://scontent.cdninstagram.com/t51.2885-15/s640x640/e15/15624670_548881701986735_8264383763249627136_n.jpg?ig_cache_key=MTQyMjkzMTczMTg0MjE3NjE3Nw%3D%3D.2'], $data['data']['photo']);
-    $this->assertEquals(1, count($data['data']['video']));
-    $this->assertEquals(['https://scontent.cdninstagram.com/t50.2886-16/15921147_1074837002642259_2269307616507199488_n.mp4'], $data['data']['video']);
-    $this->assertEquals('http://aaronparecki.com/', $data['data']['author']['url']);
-    $this->assertEquals('Aaron Parecki', $data['data']['author']['name']);
-    $this->assertEquals('https://scontent.cdninstagram.com/t51.2885-19/s320x320/14240576_268350536897085_1129715662_a.jpg', $data['data']['author']['photo']);
-  }
-
-  public function testInstagramPhotoWithPersonTag() {
-    $url = 'http://www.instagram.com/photo_with_person_tag.html';
-    $response = $this->parse(['url' => $url]);
-
-    $body = $response->getContent();
-    $this->assertEquals(200, $response->getStatusCode());
-    $data = json_decode($body, true);
-
-    $this->assertEquals(2, count($data['data']['category']));
-    $this->assertContains('https://kmikeym.com/', $data['data']['category']);
-    $this->assertArrayHasKey('https://kmikeym.com/', $data['refs']);
-    $this->assertEquals(['type'=>'card','name'=>'Mike Merrill','url'=>'https://kmikeym.com/','photo'=>'https://scontent.cdninstagram.com/t51.2885-19/s320x320/12627953_686238411518831_1544976311_a.jpg'], $data['refs']['https://kmikeym.com/']);
-  }
-
-  public function testInstagramPhotoWithVenue() {
-    $url = 'http://www.instagram.com/photo_with_venue.html';
-    $response = $this->parse(['url' => $url]);
-
-    $body = $response->getContent();
-    $this->assertEquals(200, $response->getStatusCode());
-    $data = json_decode($body, true);
-
-    $this->assertEquals(1, count($data['data']['location']));
-    $this->assertContains('https://www.instagram.com/explore/locations/109284789535230/', $data['data']['location']);
-    $this->assertArrayHasKey('https://www.instagram.com/explore/locations/109284789535230/', $data['refs']);
-    $venue = $data['refs']['https://www.instagram.com/explore/locations/109284789535230/'];
-    $this->assertEquals('XOXO Outpost', $venue['name']);
-    $this->assertEquals('45.5261002', $venue['latitude']);
-    $this->assertEquals('-122.6558081', $venue['longitude']);
-    // Setting a venue should set the timezone
-    $this->assertEquals('2016-12-10T21:48:56-08:00', $data['data']['published']);
-  }
-
 }

tests/TwitterTest.php

@@ -0,0 +1,143 @@
+<?php
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+
+class TwitterTest extends PHPUnit_Framework_TestCase {
+
+  public function setUp() {
+    $this->client = new Parse();
+    $this->client->mc = null;
+  }
+
+  private function parse($params) {
+    $request = new Request($params);
+    $response = new Response();
+    $result = $this->client->parse($request, $response);
+    $body = $result->getContent();
+    $this->assertEquals(200, $result->getStatusCode());
+    return json_decode($body, true);
+  }
+
+  private function loadTweet($id) {
+    $url = 'https://twitter.com/_/status/'.$id;
+    $json = file_get_contents(dirname(__FILE__).'/data/api.twitter.com/'.$id.'.json');
+    return [$url, $json];
+  }
+
+  public function testBasicProfileInfo() {
+    list($url, $json) = $this->loadTweet('818912506496229376');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('aaronpk dev', $data['data']['author']['name']);
+    $this->assertEquals('pkdev', $data['data']['author']['nickname']);
+    $this->assertEquals('https://aaronparecki.com/', $data['data']['author']['url']);
+    $this->assertEquals('Portland, OR', $data['data']['author']['location']);
+    $this->assertEquals('Dev account for testing Twitter things. Follow me here: https://twitter.com/aaronpk', $data['data']['author']['bio']);
+    $this->assertEquals('https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg', $data['data']['author']['photo']);
+  }
+
+  public function testBasicTestStuff() {
+    list($url, $json) = $this->loadTweet('818913630569664512');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('A tweet with a URL https://indieweb.org/ #and #some #hashtags', $data['data']['content']['text']);
+    $this->assertContains('and', $data['data']['category']);
+    $this->assertContains('some', $data['data']['category']);
+    $this->assertContains('hashtags', $data['data']['category']);
+    // Published date should be set to the timezone of the user
+    $this->assertEquals('2017-01-10T12:13:18-08:00', $data['data']['published']);
+  }
+
+  public function testTweetWithEmoji() {
+    list($url, $json) = $this->loadTweet('818943244553699328');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('Here 🎉 have an emoji', $data['data']['content']['text']);
+  }
+
+  public function testHTMLEscaping() {
+    list($url, $json) = $this->loadTweet('818928092383166465');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('Double escaping &amp; & amp', $data['data']['content']['text']);
+  }
+
+  public function testTweetWithPhoto() {
+    list($url, $json) = $this->loadTweet('818912506496229376');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('Tweet with a photo and a location', $data['data']['content']['text']);
+    $this->assertEquals('https://pbs.twimg.com/media/C11cfRJUoAI26h9.jpg', $data['data']['photo'][0]);
+  }
+
+  public function testTweetWithTwoPhotos() {
+    list($url, $json) = $this->loadTweet('818935308813103104');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('Two photos', $data['data']['content']['text']);
+    $this->assertContains('https://pbs.twimg.com/media/C11xS1wUcAAeaKF.jpg', $data['data']['photo']);
+    $this->assertContains('https://pbs.twimg.com/media/C11wtndUoAE1WfE.jpg', $data['data']['photo']);
+  }
+
+  public function testTweetWithVideo() {
+    list($url, $json) = $this->loadTweet('818913178260160512');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('Tweet with a video', $data['data']['content']['text']);
+    $this->assertEquals('https://video.twimg.com/ext_tw_video/818913089248595970/pr/vid/1280x720/qP-sDx-Q0Hs-ckVv.mp4', $data['data']['video'][0]);
+  }
+
+  public function testTweetWithLocation() {
+    list($url, $json) = $this->loadTweet('818912506496229376');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('Tweet with a photo and a location', $data['data']['content']['text']);
+    $this->assertEquals('https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json', $data['data']['location']);
+    $location = $data['refs']['https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json'];
+    $this->assertEquals('adr', $location['type']);
+    $this->assertEquals('Portland', $location['locality']);
+    $this->assertEquals('United States', $location['country-name']);
+    $this->assertEquals('Portland, OR', $location['name']);
+  }
+
+  public function testRetweet() {
+    list($url, $json) = $this->loadTweet('818913351623245824');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertArrayNotHasKey('content', $data['data']);
+    $repostOf = 'https://twitter.com/aaronpk/status/817414679131660288';
+    $this->assertEquals($repostOf, $data['data']['repost-of']);
+    $tweet = $data['refs'][$repostOf];
+    $this->assertEquals('Yeah that\'s me http://xkcd.com/1782/', $tweet['content']['text']);
+  }
+
+  public function testQuotedTweet() {
+    list($url, $json) = $this->loadTweet('818913488609251331');
+
+    $data = $this->parse(['url' => $url, 'json' => $json]);
+
+    $this->assertEquals('entry', $data['data']['type']);
+    $this->assertEquals('Quoted tweet with a #hashtag https://twitter.com/aaronpk/status/817414679131660288', $data['data']['content']['text']);
+    $tweet = $data['refs']['https://twitter.com/aaronpk/status/817414679131660288'];
+    $this->assertEquals('Yeah that\'s me http://xkcd.com/1782/', $tweet['content']['text']);
+  }
+
+}

tests/data/api.twitter.com/818912506496229376.json

@@ -0,0 +1,227 @@
+{
+  "created_at": "Tue Jan 10 20:08:50 +0000 2017",
+  "id": 818912506496229376,
+  "id_str": "818912506496229376",
+  "full_text": "Tweet with a photo and a location https://t.co/GwEzHTHlUC",
+  "truncated": false,
+  "display_text_range": [
+    0,
+    33
+  ],
+  "entities": {
+    "hashtags": [
+
+    ],
+    "symbols": [
+
+    ],
+    "user_mentions": [
+
+    ],
+    "urls": [
+
+    ],
+    "media": [
+      {
+        "id": 818912399499501570,
+        "id_str": "818912399499501570",
+        "indices": [
+          34,
+          57
+        ],
+        "media_url": "http://pbs.twimg.com/media/C11cfRJUoAI26h9.jpg",
+        "media_url_https": "https://pbs.twimg.com/media/C11cfRJUoAI26h9.jpg",
+        "url": "https://t.co/GwEzHTHlUC",
+        "display_url": "pic.twitter.com/GwEzHTHlUC",
+        "expanded_url": "https://twitter.com/pkdev/status/818912506496229376/photo/1",
+        "type": "photo",
+        "sizes": {
+          "medium": {
+            "w": 1200,
+            "h": 800,
+            "resize": "fit"
+          },
+          "small": {
+            "w": 680,
+            "h": 453,
+            "resize": "fit"
+          },
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "large": {
+            "w": 2048,
+            "h": 1365,
+            "resize": "fit"
+          }
+        }
+      }
+    ]
+  },
+  "extended_entities": {
+    "media": [
+      {
+        "id": 818912399499501570,
+        "id_str": "818912399499501570",
+        "indices": [
+          34,
+          57
+        ],
+        "media_url": "http://pbs.twimg.com/media/C11cfRJUoAI26h9.jpg",
+        "media_url_https": "https://pbs.twimg.com/media/C11cfRJUoAI26h9.jpg",
+        "url": "https://t.co/GwEzHTHlUC",
+        "display_url": "pic.twitter.com/GwEzHTHlUC",
+        "expanded_url": "https://twitter.com/pkdev/status/818912506496229376/photo/1",
+        "type": "photo",
+        "sizes": {
+          "medium": {
+            "w": 1200,
+            "h": 800,
+            "resize": "fit"
+          },
+          "small": {
+            "w": 680,
+            "h": 453,
+            "resize": "fit"
+          },
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "large": {
+            "w": 2048,
+            "h": 1365,
+            "resize": "fit"
+          }
+        }
+      }
+    ]
+  },
+  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
+  "in_reply_to_status_id": null,
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_user_id": null,
+  "in_reply_to_user_id_str": null,
+  "in_reply_to_screen_name": null,
+  "user": {
+    "id": 143883456,
+    "id_str": "143883456",
+    "name": "aaronpk dev",
+    "screen_name": "pkdev",
+    "location": "Portland, OR",
+    "description": "Dev account for testing Twitter things. Follow me here: http://t.co/DtzRLfxayu",
+    "url": "https://t.co/fXLomQaMAd",
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/fXLomQaMAd",
+            "expanded_url": "https://aaronparecki.com/",
+            "display_url": "aaronparecki.com",
+            "indices": [
+              0,
+              23
+            ]
+          }
+        ]
+      },
+      "description": {
+        "urls": [
+          {
+            "url": "http://t.co/DtzRLfxayu",
+            "expanded_url": "https://twitter.com/aaronpk",
+            "display_url": "twitter.com/aaronpk",
+            "indices": [
+              56,
+              78
+            ]
+          }
+        ]
+      }
+    },
+    "protected": true,
+    "followers_count": 4,
+    "friends_count": 1,
+    "listed_count": 1,
+    "created_at": "Fri May 14 17:47:15 +0000 2010",
+    "favourites_count": 1,
+    "utc_offset": -28800,
+    "time_zone": "Pacific Time (US & Canada)",
+    "geo_enabled": true,
+    "verified": false,
+    "statuses_count": 31,
+    "lang": "en",
+    "contributors_enabled": false,
+    "is_translator": false,
+    "is_translation_enabled": false,
+    "profile_background_color": "C0DEED",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_tile": false,
+    "profile_image_url": "http://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_link_color": "1DA1F2",
+    "profile_sidebar_border_color": "C0DEED",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "profile_text_color": "333333",
+    "profile_use_background_image": true,
+    "has_extended_profile": false,
+    "default_profile": true,
+    "default_profile_image": false,
+    "following": true,
+    "follow_request_sent": false,
+    "notifications": false,
+    "translator_type": "none"
+  },
+  "geo": null,
+  "coordinates": null,
+  "place": {
+    "id": "ac88a4f17a51c7fc",
+    "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+    "place_type": "city",
+    "name": "Portland",
+    "full_name": "Portland, OR",
+    "country_code": "US",
+    "country": "United States",
+    "contained_within": [
+
+    ],
+    "bounding_box": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -122.7900653,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.6509405
+          ],
+          [
+            -122.7900653,
+            45.6509405
+          ]
+        ]
+      ]
+    },
+    "attributes": {
+    }
+  },
+  "contributors": null,
+  "is_quote_status": false,
+  "retweet_count": 0,
+  "favorite_count": 0,
+  "favorited": false,
+  "retweeted": false,
+  "possibly_sensitive": false,
+  "possibly_sensitive_appealable": false,
+  "lang": "en"
+}

tests/data/api.twitter.com/818913178260160512.json

@@ -0,0 +1,262 @@
+{
+  "created_at": "Tue Jan 10 20:11:31 +0000 2017",
+  "id": 818913178260160512,
+  "id_str": "818913178260160512",
+  "full_text": "Tweet with a video https://t.co/6hyv5rr3FL",
+  "truncated": false,
+  "display_text_range": [
+    0,
+    18
+  ],
+  "entities": {
+    "hashtags": [
+
+    ],
+    "symbols": [
+
+    ],
+    "user_mentions": [
+
+    ],
+    "urls": [
+
+    ],
+    "media": [
+      {
+        "id": 818913089248595970,
+        "id_str": "818913089248595970",
+        "indices": [
+          19,
+          42
+        ],
+        "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/818913089248595970/pr/img/qVoEjF03Y41SKpNt.jpg",
+        "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/818913089248595970/pr/img/qVoEjF03Y41SKpNt.jpg",
+        "url": "https://t.co/6hyv5rr3FL",
+        "display_url": "pic.twitter.com/6hyv5rr3FL",
+        "expanded_url": "https://twitter.com/pkdev/status/818913178260160512/video/1",
+        "type": "photo",
+        "sizes": {
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "medium": {
+            "w": 600,
+            "h": 338,
+            "resize": "fit"
+          },
+          "small": {
+            "w": 340,
+            "h": 191,
+            "resize": "fit"
+          },
+          "large": {
+            "w": 1024,
+            "h": 576,
+            "resize": "fit"
+          }
+        }
+      }
+    ]
+  },
+  "extended_entities": {
+    "media": [
+      {
+        "id": 818913089248595970,
+        "id_str": "818913089248595970",
+        "indices": [
+          19,
+          42
+        ],
+        "media_url": "http://pbs.twimg.com/ext_tw_video_thumb/818913089248595970/pr/img/qVoEjF03Y41SKpNt.jpg",
+        "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/818913089248595970/pr/img/qVoEjF03Y41SKpNt.jpg",
+        "url": "https://t.co/6hyv5rr3FL",
+        "display_url": "pic.twitter.com/6hyv5rr3FL",
+        "expanded_url": "https://twitter.com/pkdev/status/818913178260160512/video/1",
+        "type": "video",
+        "sizes": {
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "medium": {
+            "w": 600,
+            "h": 338,
+            "resize": "fit"
+          },
+          "small": {
+            "w": 340,
+            "h": 191,
+            "resize": "fit"
+          },
+          "large": {
+            "w": 1024,
+            "h": 576,
+            "resize": "fit"
+          }
+        },
+        "video_info": {
+          "aspect_ratio": [
+            16,
+            9
+          ],
+          "duration_millis": 41534,
+          "variants": [
+            {
+              "content_type": "application/x-mpegURL",
+              "url": "https://video.twimg.com/ext_tw_video/818913089248595970/pr/pl/TrPaTlyUsAN8GjxN.m3u8"
+            },
+            {
+              "bitrate": 320000,
+              "content_type": "video/mp4",
+              "url": "https://video.twimg.com/ext_tw_video/818913089248595970/pr/vid/320x180/XMltLv_V-HjjJw3B.mp4"
+            },
+            {
+              "bitrate": 2176000,
+              "content_type": "video/mp4",
+              "url": "https://video.twimg.com/ext_tw_video/818913089248595970/pr/vid/1280x720/qP-sDx-Q0Hs-ckVv.mp4"
+            },
+            {
+              "content_type": "application/dash+xml",
+              "url": "https://video.twimg.com/ext_tw_video/818913089248595970/pr/pl/TrPaTlyUsAN8GjxN.mpd"
+            },
+            {
+              "bitrate": 832000,
+              "content_type": "video/mp4",
+              "url": "https://video.twimg.com/ext_tw_video/818913089248595970/pr/vid/640x360/1oP83JGgjXpDd4WY.mp4"
+            }
+          ]
+        },
+        "additional_media_info": {
+          "monetizable": false
+        }
+      }
+    ]
+  },
+  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
+  "in_reply_to_status_id": null,
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_user_id": null,
+  "in_reply_to_user_id_str": null,
+  "in_reply_to_screen_name": null,
+  "user": {
+    "id": 143883456,
+    "id_str": "143883456",
+    "name": "aaronpk dev",
+    "screen_name": "pkdev",
+    "location": "Portland, OR",
+    "description": "Dev account for testing Twitter things. Follow me here: http://t.co/DtzRLfxayu",
+    "url": "https://t.co/fXLomQaMAd",
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/fXLomQaMAd",
+            "expanded_url": "https://aaronparecki.com/",
+            "display_url": "aaronparecki.com",
+            "indices": [
+              0,
+              23
+            ]
+          }
+        ]
+      },
+      "description": {
+        "urls": [
+          {
+            "url": "http://t.co/DtzRLfxayu",
+            "expanded_url": "https://twitter.com/aaronpk",
+            "display_url": "twitter.com/aaronpk",
+            "indices": [
+              56,
+              78
+            ]
+          }
+        ]
+      }
+    },
+    "protected": true,
+    "followers_count": 4,
+    "friends_count": 1,
+    "listed_count": 1,
+    "created_at": "Fri May 14 17:47:15 +0000 2010",
+    "favourites_count": 1,
+    "utc_offset": -28800,
+    "time_zone": "Pacific Time (US & Canada)",
+    "geo_enabled": true,
+    "verified": false,
+    "statuses_count": 36,
+    "lang": "en",
+    "contributors_enabled": false,
+    "is_translator": false,
+    "is_translation_enabled": false,
+    "profile_background_color": "C0DEED",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_tile": false,
+    "profile_image_url": "http://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_link_color": "1DA1F2",
+    "profile_sidebar_border_color": "C0DEED",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "profile_text_color": "333333",
+    "profile_use_background_image": true,
+    "has_extended_profile": true,
+    "default_profile": true,
+    "default_profile_image": false,
+    "following": true,
+    "follow_request_sent": false,
+    "notifications": false,
+    "translator_type": "none"
+  },
+  "geo": null,
+  "coordinates": null,
+  "place": {
+    "id": "ac88a4f17a51c7fc",
+    "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+    "place_type": "city",
+    "name": "Portland",
+    "full_name": "Portland, OR",
+    "country_code": "US",
+    "country": "United States",
+    "contained_within": [
+
+    ],
+    "bounding_box": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -122.7900653,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.6509405
+          ],
+          [
+            -122.7900653,
+            45.6509405
+          ]
+        ]
+      ]
+    },
+    "attributes": {
+    }
+  },
+  "contributors": null,
+  "is_quote_status": false,
+  "retweet_count": 0,
+  "favorite_count": 0,
+  "favorited": false,
+  "retweeted": false,
+  "possibly_sensitive": false,
+  "possibly_sensitive_appealable": false,
+  "lang": "en"
+}

tests/data/api.twitter.com/818913351623245824.json

@@ -0,0 +1,482 @@
+{
+  "created_at": "Tue Jan 10 20:12:12 +0000 2017",
+  "id": 818913351623245824,
+  "id_str": "818913351623245824",
+  "full_text": "RT @aaronpk: Yeah that's me https://t.co/6ZjcRmb3ec https://t.co/n0k56i1nSl",
+  "truncated": false,
+  "display_text_range": [
+    0,
+    75
+  ],
+  "entities": {
+    "hashtags": [
+
+    ],
+    "symbols": [
+
+    ],
+    "user_mentions": [
+      {
+        "screen_name": "aaronpk",
+        "name": "Aaron Parecki",
+        "id": 14447132,
+        "id_str": "14447132",
+        "indices": [
+          3,
+          11
+        ]
+      }
+    ],
+    "urls": [
+      {
+        "url": "https://t.co/6ZjcRmb3ec",
+        "expanded_url": "http://xkcd.com/1782/",
+        "display_url": "xkcd.com/1782/",
+        "indices": [
+          28,
+          51
+        ]
+      }
+    ],
+    "media": [
+      {
+        "id": 817414678586372096,
+        "id_str": "817414678586372096",
+        "indices": [
+          52,
+          75
+        ],
+        "media_url": "http://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+        "media_url_https": "https://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+        "url": "https://t.co/n0k56i1nSl",
+        "display_url": "pic.twitter.com/n0k56i1nSl",
+        "expanded_url": "https://twitter.com/aaronpk/status/817414679131660288/photo/1",
+        "type": "photo",
+        "sizes": {
+          "small": {
+            "w": 680,
+            "h": 290,
+            "resize": "fit"
+          },
+          "large": {
+            "w": 700,
+            "h": 299,
+            "resize": "fit"
+          },
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "medium": {
+            "w": 700,
+            "h": 299,
+            "resize": "fit"
+          }
+        },
+        "source_status_id": 817414679131660288,
+        "source_status_id_str": "817414679131660288",
+        "source_user_id": 14447132,
+        "source_user_id_str": "14447132"
+      }
+    ]
+  },
+  "extended_entities": {
+    "media": [
+      {
+        "id": 817414678586372096,
+        "id_str": "817414678586372096",
+        "indices": [
+          52,
+          75
+        ],
+        "media_url": "http://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+        "media_url_https": "https://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+        "url": "https://t.co/n0k56i1nSl",
+        "display_url": "pic.twitter.com/n0k56i1nSl",
+        "expanded_url": "https://twitter.com/aaronpk/status/817414679131660288/photo/1",
+        "type": "photo",
+        "sizes": {
+          "small": {
+            "w": 680,
+            "h": 290,
+            "resize": "fit"
+          },
+          "large": {
+            "w": 700,
+            "h": 299,
+            "resize": "fit"
+          },
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "medium": {
+            "w": 700,
+            "h": 299,
+            "resize": "fit"
+          }
+        },
+        "source_status_id": 817414679131660288,
+        "source_status_id_str": "817414679131660288",
+        "source_user_id": 14447132,
+        "source_user_id_str": "14447132"
+      }
+    ]
+  },
+  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
+  "in_reply_to_status_id": null,
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_user_id": null,
+  "in_reply_to_user_id_str": null,
+  "in_reply_to_screen_name": null,
+  "user": {
+    "id": 143883456,
+    "id_str": "143883456",
+    "name": "aaronpk dev",
+    "screen_name": "pkdev",
+    "location": "Portland, OR",
+    "description": "Dev account for testing Twitter things. Follow me here: http://t.co/DtzRLfxayu",
+    "url": "https://t.co/fXLomQaMAd",
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/fXLomQaMAd",
+            "expanded_url": "https://aaronparecki.com/",
+            "display_url": "aaronparecki.com",
+            "indices": [
+              0,
+              23
+            ]
+          }
+        ]
+      },
+      "description": {
+        "urls": [
+          {
+            "url": "http://t.co/DtzRLfxayu",
+            "expanded_url": "https://twitter.com/aaronpk",
+            "display_url": "twitter.com/aaronpk",
+            "indices": [
+              56,
+              78
+            ]
+          }
+        ]
+      }
+    },
+    "protected": true,
+    "followers_count": 4,
+    "friends_count": 1,
+    "listed_count": 1,
+    "created_at": "Fri May 14 17:47:15 +0000 2010",
+    "favourites_count": 1,
+    "utc_offset": -28800,
+    "time_zone": "Pacific Time (US & Canada)",
+    "geo_enabled": true,
+    "verified": false,
+    "statuses_count": 37,
+    "lang": "en",
+    "contributors_enabled": false,
+    "is_translator": false,
+    "is_translation_enabled": false,
+    "profile_background_color": "C0DEED",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_tile": false,
+    "profile_image_url": "http://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_link_color": "1DA1F2",
+    "profile_sidebar_border_color": "C0DEED",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "profile_text_color": "333333",
+    "profile_use_background_image": true,
+    "has_extended_profile": true,
+    "default_profile": true,
+    "default_profile_image": false,
+    "following": true,
+    "follow_request_sent": false,
+    "notifications": false,
+    "translator_type": "none"
+  },
+  "geo": null,
+  "coordinates": null,
+  "place": null,
+  "contributors": null,
+  "retweeted_status": {
+    "created_at": "Fri Jan 06 16:57:00 +0000 2017",
+    "id": 817414679131660288,
+    "id_str": "817414679131660288",
+    "full_text": "Yeah that's me https://t.co/6ZjcRmb3ec https://t.co/n0k56i1nSl",
+    "truncated": false,
+    "display_text_range": [
+      0,
+      38
+    ],
+    "entities": {
+      "hashtags": [
+
+      ],
+      "symbols": [
+
+      ],
+      "user_mentions": [
+
+      ],
+      "urls": [
+        {
+          "url": "https://t.co/6ZjcRmb3ec",
+          "expanded_url": "http://xkcd.com/1782/",
+          "display_url": "xkcd.com/1782/",
+          "indices": [
+            15,
+            38
+          ]
+        }
+      ],
+      "media": [
+        {
+          "id": 817414678586372096,
+          "id_str": "817414678586372096",
+          "indices": [
+            39,
+            62
+          ],
+          "media_url": "http://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+          "media_url_https": "https://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+          "url": "https://t.co/n0k56i1nSl",
+          "display_url": "pic.twitter.com/n0k56i1nSl",
+          "expanded_url": "https://twitter.com/aaronpk/status/817414679131660288/photo/1",
+          "type": "photo",
+          "sizes": {
+            "small": {
+              "w": 680,
+              "h": 290,
+              "resize": "fit"
+            },
+            "large": {
+              "w": 700,
+              "h": 299,
+              "resize": "fit"
+            },
+            "thumb": {
+              "w": 150,
+              "h": 150,
+              "resize": "crop"
+            },
+            "medium": {
+              "w": 700,
+              "h": 299,
+              "resize": "fit"
+            }
+          }
+        }
+      ]
+    },
+    "extended_entities": {
+      "media": [
+        {
+          "id": 817414678586372096,
+          "id_str": "817414678586372096",
+          "indices": [
+            39,
+            62
+          ],
+          "media_url": "http://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+          "media_url_https": "https://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+          "url": "https://t.co/n0k56i1nSl",
+          "display_url": "pic.twitter.com/n0k56i1nSl",
+          "expanded_url": "https://twitter.com/aaronpk/status/817414679131660288/photo/1",
+          "type": "photo",
+          "sizes": {
+            "small": {
+              "w": 680,
+              "h": 290,
+              "resize": "fit"
+            },
+            "large": {
+              "w": 700,
+              "h": 299,
+              "resize": "fit"
+            },
+            "thumb": {
+              "w": 150,
+              "h": 150,
+              "resize": "crop"
+            },
+            "medium": {
+              "w": 700,
+              "h": 299,
+              "resize": "fit"
+            }
+          }
+        }
+      ]
+    },
+    "source": "<a href=\"https://silopub.p3k.io\" rel=\"nofollow\">Silo Pub for p3k</a>",
+    "in_reply_to_status_id": null,
+    "in_reply_to_status_id_str": null,
+    "in_reply_to_user_id": null,
+    "in_reply_to_user_id_str": null,
+    "in_reply_to_screen_name": null,
+    "user": {
+      "id": 14447132,
+      "id_str": "14447132",
+      "name": "Aaron Parecki",
+      "screen_name": "aaronpk",
+      "location": "Portland, OR",
+      "description": "Cofounder of #indieweb/@indiewebcamp • I maintain https://t.co/zBgRc3oyhG • Editor of https://t.co/JxW2Ws0Ctv and https://t.co/bjsh7VQ4ZI • @W7APK",
+      "url": "https://t.co/LvwC5LPQlU",
+      "entities": {
+        "url": {
+          "urls": [
+            {
+              "url": "https://t.co/LvwC5LPQlU",
+              "expanded_url": "http://aaronparecki.com",
+              "display_url": "aaronparecki.com",
+              "indices": [
+                0,
+                23
+              ]
+            }
+          ]
+        },
+        "description": {
+          "urls": [
+            {
+              "url": "https://t.co/zBgRc3oyhG",
+              "expanded_url": "http://oauth.net",
+              "display_url": "oauth.net",
+              "indices": [
+                50,
+                73
+              ]
+            },
+            {
+              "url": "https://t.co/JxW2Ws0Ctv",
+              "expanded_url": "http://w3.org/TR/micropub",
+              "display_url": "w3.org/TR/micropub",
+              "indices": [
+                86,
+                109
+              ]
+            },
+            {
+              "url": "https://t.co/bjsh7VQ4ZI",
+              "expanded_url": "http://w3.org/TR/webmention",
+              "display_url": "w3.org/TR/webmention",
+              "indices": [
+                114,
+                137
+              ]
+            }
+          ]
+        }
+      },
+      "protected": false,
+      "followers_count": 3379,
+      "friends_count": 1036,
+      "listed_count": 346,
+      "created_at": "Sat Apr 19 22:38:15 +0000 2008",
+      "favourites_count": 3703,
+      "utc_offset": -28800,
+      "time_zone": "Pacific Time (US & Canada)",
+      "geo_enabled": true,
+      "verified": true,
+      "statuses_count": 6675,
+      "lang": "en",
+      "contributors_enabled": false,
+      "is_translator": false,
+      "is_translation_enabled": false,
+      "profile_background_color": "7A9AAF",
+      "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/185835062/4786064324_b7049fbec8_b.jpg",
+      "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/185835062/4786064324_b7049fbec8_b.jpg",
+      "profile_background_tile": true,
+      "profile_image_url": "http://pbs.twimg.com/profile_images/775755455188512768/CA3YxGa4_normal.jpg",
+      "profile_image_url_https": "https://pbs.twimg.com/profile_images/775755455188512768/CA3YxGa4_normal.jpg",
+      "profile_banner_url": "https://pbs.twimg.com/profile_banners/14447132/1398201184",
+      "profile_link_color": "FF5900",
+      "profile_sidebar_border_color": "87BC44",
+      "profile_sidebar_fill_color": "94C8FF",
+      "profile_text_color": "000000",
+      "profile_use_background_image": true,
+      "has_extended_profile": true,
+      "default_profile": false,
+      "default_profile_image": false,
+      "following": false,
+      "follow_request_sent": false,
+      "notifications": false,
+      "translator_type": "none"
+    },
+    "geo": {
+      "type": "Point",
+      "coordinates": [
+        45.53553,
+        -122.62139
+      ]
+    },
+    "coordinates": {
+      "type": "Point",
+      "coordinates": [
+        -122.62139,
+        45.53553
+      ]
+    },
+    "place": {
+      "id": "ac88a4f17a51c7fc",
+      "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+      "place_type": "city",
+      "name": "Portland",
+      "full_name": "Portland, OR",
+      "country_code": "US",
+      "country": "United States",
+      "contained_within": [
+
+      ],
+      "bounding_box": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [
+              -122.7900653,
+              45.421863
+            ],
+            [
+              -122.471751,
+              45.421863
+            ],
+            [
+              -122.471751,
+              45.6509405
+            ],
+            [
+              -122.7900653,
+              45.6509405
+            ]
+          ]
+        ]
+      },
+      "attributes": {
+      }
+    },
+    "contributors": null,
+    "is_quote_status": false,
+    "retweet_count": 3,
+    "favorite_count": 6,
+    "favorited": false,
+    "retweeted": false,
+    "possibly_sensitive": false,
+    "possibly_sensitive_appealable": false,
+    "lang": "en"
+  },
+  "is_quote_status": false,
+  "retweet_count": 3,
+  "favorite_count": 0,
+  "favorited": false,
+  "retweeted": false,
+  "possibly_sensitive": false,
+  "possibly_sensitive_appealable": false,
+  "lang": "en"
+}

tests/data/api.twitter.com/818913488609251331.json

@@ -0,0 +1,431 @@
+{
+  "created_at": "Tue Jan 10 20:12:45 +0000 2017",
+  "id": 818913488609251331,
+  "id_str": "818913488609251331",
+  "full_text": "Quoted tweet with a #hashtag https://t.co/m8RAfr0S3e",
+  "truncated": false,
+  "display_text_range": [
+    0,
+    52
+  ],
+  "entities": {
+    "hashtags": [
+      {
+        "text": "hashtag",
+        "indices": [
+          20,
+          28
+        ]
+      }
+    ],
+    "symbols": [
+
+    ],
+    "user_mentions": [
+
+    ],
+    "urls": [
+      {
+        "url": "https://t.co/m8RAfr0S3e",
+        "expanded_url": "https://twitter.com/aaronpk/status/817414679131660288",
+        "display_url": "twitter.com/aaronpk/status…",
+        "indices": [
+          29,
+          52
+        ]
+      }
+    ]
+  },
+  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
+  "in_reply_to_status_id": null,
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_user_id": null,
+  "in_reply_to_user_id_str": null,
+  "in_reply_to_screen_name": null,
+  "user": {
+    "id": 143883456,
+    "id_str": "143883456",
+    "name": "aaronpk dev",
+    "screen_name": "pkdev",
+    "location": "Portland, OR",
+    "description": "Dev account for testing Twitter things. Follow me here: http://t.co/DtzRLfxayu",
+    "url": "https://t.co/fXLomQaMAd",
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/fXLomQaMAd",
+            "expanded_url": "https://aaronparecki.com/",
+            "display_url": "aaronparecki.com",
+            "indices": [
+              0,
+              23
+            ]
+          }
+        ]
+      },
+      "description": {
+        "urls": [
+          {
+            "url": "http://t.co/DtzRLfxayu",
+            "expanded_url": "https://twitter.com/aaronpk",
+            "display_url": "twitter.com/aaronpk",
+            "indices": [
+              56,
+              78
+            ]
+          }
+        ]
+      }
+    },
+    "protected": true,
+    "followers_count": 4,
+    "friends_count": 1,
+    "listed_count": 1,
+    "created_at": "Fri May 14 17:47:15 +0000 2010",
+    "favourites_count": 1,
+    "utc_offset": -28800,
+    "time_zone": "Pacific Time (US & Canada)",
+    "geo_enabled": true,
+    "verified": false,
+    "statuses_count": 37,
+    "lang": "en",
+    "contributors_enabled": false,
+    "is_translator": false,
+    "is_translation_enabled": false,
+    "profile_background_color": "C0DEED",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_tile": false,
+    "profile_image_url": "http://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_link_color": "1DA1F2",
+    "profile_sidebar_border_color": "C0DEED",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "profile_text_color": "333333",
+    "profile_use_background_image": true,
+    "has_extended_profile": true,
+    "default_profile": true,
+    "default_profile_image": false,
+    "following": true,
+    "follow_request_sent": false,
+    "notifications": false,
+    "translator_type": "none"
+  },
+  "geo": null,
+  "coordinates": null,
+  "place": {
+    "id": "ac88a4f17a51c7fc",
+    "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+    "place_type": "city",
+    "name": "Portland",
+    "full_name": "Portland, OR",
+    "country_code": "US",
+    "country": "United States",
+    "contained_within": [
+
+    ],
+    "bounding_box": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -122.7900653,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.6509405
+          ],
+          [
+            -122.7900653,
+            45.6509405
+          ]
+        ]
+      ]
+    },
+    "attributes": {
+    }
+  },
+  "contributors": null,
+  "is_quote_status": true,
+  "quoted_status_id": 817414679131660288,
+  "quoted_status_id_str": "817414679131660288",
+  "quoted_status": {
+    "created_at": "Fri Jan 06 16:57:00 +0000 2017",
+    "id": 817414679131660288,
+    "id_str": "817414679131660288",
+    "full_text": "Yeah that's me https://t.co/6ZjcRmb3ec https://t.co/n0k56i1nSl",
+    "truncated": false,
+    "display_text_range": [
+      0,
+      38
+    ],
+    "entities": {
+      "hashtags": [
+
+      ],
+      "symbols": [
+
+      ],
+      "user_mentions": [
+
+      ],
+      "urls": [
+        {
+          "url": "https://t.co/6ZjcRmb3ec",
+          "expanded_url": "http://xkcd.com/1782/",
+          "display_url": "xkcd.com/1782/",
+          "indices": [
+            15,
+            38
+          ]
+        }
+      ],
+      "media": [
+        {
+          "id": 817414678586372096,
+          "id_str": "817414678586372096",
+          "indices": [
+            39,
+            62
+          ],
+          "media_url": "http://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+          "media_url_https": "https://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+          "url": "https://t.co/n0k56i1nSl",
+          "display_url": "pic.twitter.com/n0k56i1nSl",
+          "expanded_url": "https://twitter.com/aaronpk/status/817414679131660288/photo/1",
+          "type": "photo",
+          "sizes": {
+            "small": {
+              "w": 680,
+              "h": 290,
+              "resize": "fit"
+            },
+            "large": {
+              "w": 700,
+              "h": 299,
+              "resize": "fit"
+            },
+            "thumb": {
+              "w": 150,
+              "h": 150,
+              "resize": "crop"
+            },
+            "medium": {
+              "w": 700,
+              "h": 299,
+              "resize": "fit"
+            }
+          }
+        }
+      ]
+    },
+    "extended_entities": {
+      "media": [
+        {
+          "id": 817414678586372096,
+          "id_str": "817414678586372096",
+          "indices": [
+            39,
+            62
+          ],
+          "media_url": "http://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+          "media_url_https": "https://pbs.twimg.com/media/C1gKUb9UsAAbi2R.jpg",
+          "url": "https://t.co/n0k56i1nSl",
+          "display_url": "pic.twitter.com/n0k56i1nSl",
+          "expanded_url": "https://twitter.com/aaronpk/status/817414679131660288/photo/1",
+          "type": "photo",
+          "sizes": {
+            "small": {
+              "w": 680,
+              "h": 290,
+              "resize": "fit"
+            },
+            "large": {
+              "w": 700,
+              "h": 299,
+              "resize": "fit"
+            },
+            "thumb": {
+              "w": 150,
+              "h": 150,
+              "resize": "crop"
+            },
+            "medium": {
+              "w": 700,
+              "h": 299,
+              "resize": "fit"
+            }
+          }
+        }
+      ]
+    },
+    "source": "<a href=\"https://silopub.p3k.io\" rel=\"nofollow\">Silo Pub for p3k</a>",
+    "in_reply_to_status_id": null,
+    "in_reply_to_status_id_str": null,
+    "in_reply_to_user_id": null,
+    "in_reply_to_user_id_str": null,
+    "in_reply_to_screen_name": null,
+    "user": {
+      "id": 14447132,
+      "id_str": "14447132",
+      "name": "Aaron Parecki",
+      "screen_name": "aaronpk",
+      "location": "Portland, OR",
+      "description": "Cofounder of #indieweb/@indiewebcamp • I maintain https://t.co/zBgRc3oyhG • Editor of https://t.co/JxW2Ws0Ctv and https://t.co/bjsh7VQ4ZI • @W7APK",
+      "url": "https://t.co/LvwC5LPQlU",
+      "entities": {
+        "url": {
+          "urls": [
+            {
+              "url": "https://t.co/LvwC5LPQlU",
+              "expanded_url": "http://aaronparecki.com",
+              "display_url": "aaronparecki.com",
+              "indices": [
+                0,
+                23
+              ]
+            }
+          ]
+        },
+        "description": {
+          "urls": [
+            {
+              "url": "https://t.co/zBgRc3oyhG",
+              "expanded_url": "http://oauth.net",
+              "display_url": "oauth.net",
+              "indices": [
+                50,
+                73
+              ]
+            },
+            {
+              "url": "https://t.co/JxW2Ws0Ctv",
+              "expanded_url": "http://w3.org/TR/micropub",
+              "display_url": "w3.org/TR/micropub",
+              "indices": [
+                86,
+                109
+              ]
+            },
+            {
+              "url": "https://t.co/bjsh7VQ4ZI",
+              "expanded_url": "http://w3.org/TR/webmention",
+              "display_url": "w3.org/TR/webmention",
+              "indices": [
+                114,
+                137
+              ]
+            }
+          ]
+        }
+      },
+      "protected": false,
+      "followers_count": 3379,
+      "friends_count": 1036,
+      "listed_count": 346,
+      "created_at": "Sat Apr 19 22:38:15 +0000 2008",
+      "favourites_count": 3703,
+      "utc_offset": -28800,
+      "time_zone": "Pacific Time (US & Canada)",
+      "geo_enabled": true,
+      "verified": true,
+      "statuses_count": 6675,
+      "lang": "en",
+      "contributors_enabled": false,
+      "is_translator": false,
+      "is_translation_enabled": false,
+      "profile_background_color": "7A9AAF",
+      "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/185835062/4786064324_b7049fbec8_b.jpg",
+      "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/185835062/4786064324_b7049fbec8_b.jpg",
+      "profile_background_tile": true,
+      "profile_image_url": "http://pbs.twimg.com/profile_images/775755455188512768/CA3YxGa4_normal.jpg",
+      "profile_image_url_https": "https://pbs.twimg.com/profile_images/775755455188512768/CA3YxGa4_normal.jpg",
+      "profile_banner_url": "https://pbs.twimg.com/profile_banners/14447132/1398201184",
+      "profile_link_color": "FF5900",
+      "profile_sidebar_border_color": "87BC44",
+      "profile_sidebar_fill_color": "94C8FF",
+      "profile_text_color": "000000",
+      "profile_use_background_image": true,
+      "has_extended_profile": true,
+      "default_profile": false,
+      "default_profile_image": false,
+      "following": false,
+      "follow_request_sent": false,
+      "notifications": false,
+      "translator_type": "none"
+    },
+    "geo": {
+      "type": "Point",
+      "coordinates": [
+        45.53553,
+        -122.62139
+      ]
+    },
+    "coordinates": {
+      "type": "Point",
+      "coordinates": [
+        -122.62139,
+        45.53553
+      ]
+    },
+    "place": {
+      "id": "ac88a4f17a51c7fc",
+      "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+      "place_type": "city",
+      "name": "Portland",
+      "full_name": "Portland, OR",
+      "country_code": "US",
+      "country": "United States",
+      "contained_within": [
+
+      ],
+      "bounding_box": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [
+              -122.7900653,
+              45.421863
+            ],
+            [
+              -122.471751,
+              45.421863
+            ],
+            [
+              -122.471751,
+              45.6509405
+            ],
+            [
+              -122.7900653,
+              45.6509405
+            ]
+          ]
+        ]
+      },
+      "attributes": {
+      }
+    },
+    "contributors": null,
+    "is_quote_status": false,
+    "retweet_count": 3,
+    "favorite_count": 6,
+    "favorited": false,
+    "retweeted": false,
+    "possibly_sensitive": false,
+    "possibly_sensitive_appealable": false,
+    "lang": "en"
+  },
+  "retweet_count": 0,
+  "favorite_count": 0,
+  "favorited": false,
+  "retweeted": false,
+  "possibly_sensitive": false,
+  "possibly_sensitive_appealable": false,
+  "lang": "en"
+}

tests/data/api.twitter.com/818913630569664512.json

@@ -0,0 +1,177 @@
+{
+  "created_at": "Tue Jan 10 20:13:18 +0000 2017",
+  "id": 818913630569664512,
+  "id_str": "818913630569664512",
+  "full_text": "A tweet with a URL https://t.co/sVq9EOlcNs #and #some #hashtags",
+  "truncated": false,
+  "display_text_range": [
+    0,
+    63
+  ],
+  "entities": {
+    "hashtags": [
+      {
+        "text": "and",
+        "indices": [
+          43,
+          47
+        ]
+      },
+      {
+        "text": "some",
+        "indices": [
+          48,
+          53
+        ]
+      },
+      {
+        "text": "hashtags",
+        "indices": [
+          54,
+          63
+        ]
+      }
+    ],
+    "symbols": [
+
+    ],
+    "user_mentions": [
+
+    ],
+    "urls": [
+      {
+        "url": "https://t.co/sVq9EOlcNs",
+        "expanded_url": "https://indieweb.org/",
+        "display_url": "indieweb.org",
+        "indices": [
+          19,
+          42
+        ]
+      }
+    ]
+  },
+  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
+  "in_reply_to_status_id": null,
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_user_id": null,
+  "in_reply_to_user_id_str": null,
+  "in_reply_to_screen_name": null,
+  "user": {
+    "id": 143883456,
+    "id_str": "143883456",
+    "name": "aaronpk dev",
+    "screen_name": "pkdev",
+    "location": "Portland, OR",
+    "description": "Dev account for testing Twitter things. Follow me here: http://t.co/DtzRLfxayu",
+    "url": "https://t.co/fXLomQaMAd",
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/fXLomQaMAd",
+            "expanded_url": "https://aaronparecki.com/",
+            "display_url": "aaronparecki.com",
+            "indices": [
+              0,
+              23
+            ]
+          }
+        ]
+      },
+      "description": {
+        "urls": [
+          {
+            "url": "http://t.co/DtzRLfxayu",
+            "expanded_url": "https://twitter.com/aaronpk",
+            "display_url": "twitter.com/aaronpk",
+            "indices": [
+              56,
+              78
+            ]
+          }
+        ]
+      }
+    },
+    "protected": true,
+    "followers_count": 4,
+    "friends_count": 1,
+    "listed_count": 1,
+    "created_at": "Fri May 14 17:47:15 +0000 2010",
+    "favourites_count": 1,
+    "utc_offset": -28800,
+    "time_zone": "Pacific Time (US & Canada)",
+    "geo_enabled": true,
+    "verified": false,
+    "statuses_count": 35,
+    "lang": "en",
+    "contributors_enabled": false,
+    "is_translator": false,
+    "is_translation_enabled": false,
+    "profile_background_color": "C0DEED",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_tile": false,
+    "profile_image_url": "http://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_link_color": "1DA1F2",
+    "profile_sidebar_border_color": "C0DEED",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "profile_text_color": "333333",
+    "profile_use_background_image": true,
+    "has_extended_profile": false,
+    "default_profile": true,
+    "default_profile_image": false,
+    "following": true,
+    "follow_request_sent": false,
+    "notifications": false,
+    "translator_type": "none"
+  },
+  "geo": null,
+  "coordinates": null,
+  "place": {
+    "id": "ac88a4f17a51c7fc",
+    "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+    "place_type": "city",
+    "name": "Portland",
+    "full_name": "Portland, OR",
+    "country_code": "US",
+    "country": "United States",
+    "contained_within": [
+
+    ],
+    "bounding_box": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -122.7900653,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.6509405
+          ],
+          [
+            -122.7900653,
+            45.6509405
+          ]
+        ]
+      ]
+    },
+    "attributes": {
+    }
+  },
+  "contributors": null,
+  "is_quote_status": false,
+  "retweet_count": 0,
+  "favorite_count": 0,
+  "favorited": false,
+  "retweeted": false,
+  "possibly_sensitive": false,
+  "possibly_sensitive_appealable": false,
+  "lang": "en"
+}

tests/data/api.twitter.com/818928092383166465.json

@@ -0,0 +1,147 @@
+{
+  "created_at": "Tue Jan 10 21:10:46 +0000 2017",
+  "id": 818928092383166465,
+  "id_str": "818928092383166465",
+  "full_text": "Double escaping & & amp",
+  "truncated": false,
+  "display_text_range": [
+    0,
+    35
+  ],
+  "entities": {
+    "hashtags": [
+
+    ],
+    "symbols": [
+
+    ],
+    "user_mentions": [
+
+    ],
+    "urls": [
+
+    ]
+  },
+  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
+  "in_reply_to_status_id": null,
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_user_id": null,
+  "in_reply_to_user_id_str": null,
+  "in_reply_to_screen_name": null,
+  "user": {
+    "id": 143883456,
+    "id_str": "143883456",
+    "name": "aaronpk dev",
+    "screen_name": "pkdev",
+    "location": "Portland, OR",
+    "description": "Dev account for testing Twitter things. Follow me here: http://t.co/DtzRLfxayu",
+    "url": "https://t.co/fXLomQaMAd",
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/fXLomQaMAd",
+            "expanded_url": "https://aaronparecki.com/",
+            "display_url": "aaronparecki.com",
+            "indices": [
+              0,
+              23
+            ]
+          }
+        ]
+      },
+      "description": {
+        "urls": [
+          {
+            "url": "http://t.co/DtzRLfxayu",
+            "expanded_url": "https://twitter.com/aaronpk",
+            "display_url": "twitter.com/aaronpk",
+            "indices": [
+              56,
+              78
+            ]
+          }
+        ]
+      }
+    },
+    "protected": true,
+    "followers_count": 4,
+    "friends_count": 1,
+    "listed_count": 1,
+    "created_at": "Fri May 14 17:47:15 +0000 2010",
+    "favourites_count": 1,
+    "utc_offset": -28800,
+    "time_zone": "Pacific Time (US & Canada)",
+    "geo_enabled": true,
+    "verified": false,
+    "statuses_count": 36,
+    "lang": "en",
+    "contributors_enabled": false,
+    "is_translator": false,
+    "is_translation_enabled": false,
+    "profile_background_color": "C0DEED",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_tile": false,
+    "profile_image_url": "http://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_link_color": "1DA1F2",
+    "profile_sidebar_border_color": "C0DEED",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "profile_text_color": "333333",
+    "profile_use_background_image": true,
+    "has_extended_profile": true,
+    "default_profile": true,
+    "default_profile_image": false,
+    "following": true,
+    "follow_request_sent": false,
+    "notifications": false,
+    "translator_type": "none"
+  },
+  "geo": null,
+  "coordinates": null,
+  "place": {
+    "id": "ac88a4f17a51c7fc",
+    "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+    "place_type": "city",
+    "name": "Portland",
+    "full_name": "Portland, OR",
+    "country_code": "US",
+    "country": "United States",
+    "contained_within": [
+
+    ],
+    "bounding_box": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -122.7900653,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.6509405
+          ],
+          [
+            -122.7900653,
+            45.6509405
+          ]
+        ]
+      ]
+    },
+    "attributes": {
+    }
+  },
+  "contributors": null,
+  "is_quote_status": false,
+  "retweet_count": 0,
+  "favorite_count": 0,
+  "favorited": false,
+  "retweeted": false,
+  "lang": "en"
+}

tests/data/api.twitter.com/818935308813103104.json

@@ -0,0 +1,263 @@
+{
+  "created_at": "Tue Jan 10 21:39:27 +0000 2017",
+  "id": 818935308813103104,
+  "id_str": "818935308813103104",
+  "full_text": "Two photos https://t.co/esnJCdKVol",
+  "truncated": false,
+  "display_text_range": [
+    0,
+    10
+  ],
+  "entities": {
+    "hashtags": [
+
+    ],
+    "symbols": [
+
+    ],
+    "user_mentions": [
+
+    ],
+    "urls": [
+
+    ],
+    "media": [
+      {
+        "id": 818934636239691777,
+        "id_str": "818934636239691777",
+        "indices": [
+          11,
+          34
+        ],
+        "media_url": "http://pbs.twimg.com/media/C11wtndUoAE1WfE.jpg",
+        "media_url_https": "https://pbs.twimg.com/media/C11wtndUoAE1WfE.jpg",
+        "url": "https://t.co/esnJCdKVol",
+        "display_url": "pic.twitter.com/esnJCdKVol",
+        "expanded_url": "https://twitter.com/pkdev/status/818935308813103104/photo/1",
+        "type": "photo",
+        "sizes": {
+          "medium": {
+            "w": 1200,
+            "h": 800,
+            "resize": "fit"
+          },
+          "small": {
+            "w": 680,
+            "h": 453,
+            "resize": "fit"
+          },
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "large": {
+            "w": 2048,
+            "h": 1365,
+            "resize": "fit"
+          }
+        }
+      }
+    ]
+  },
+  "extended_entities": {
+    "media": [
+      {
+        "id": 818934636239691777,
+        "id_str": "818934636239691777",
+        "indices": [
+          11,
+          34
+        ],
+        "media_url": "http://pbs.twimg.com/media/C11wtndUoAE1WfE.jpg",
+        "media_url_https": "https://pbs.twimg.com/media/C11wtndUoAE1WfE.jpg",
+        "url": "https://t.co/esnJCdKVol",
+        "display_url": "pic.twitter.com/esnJCdKVol",
+        "expanded_url": "https://twitter.com/pkdev/status/818935308813103104/photo/1",
+        "type": "photo",
+        "sizes": {
+          "medium": {
+            "w": 1200,
+            "h": 800,
+            "resize": "fit"
+          },
+          "small": {
+            "w": 680,
+            "h": 453,
+            "resize": "fit"
+          },
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "large": {
+            "w": 2048,
+            "h": 1365,
+            "resize": "fit"
+          }
+        }
+      },
+      {
+        "id": 818935275732627456,
+        "id_str": "818935275732627456",
+        "indices": [
+          11,
+          34
+        ],
+        "media_url": "http://pbs.twimg.com/media/C11xS1wUcAAeaKF.jpg",
+        "media_url_https": "https://pbs.twimg.com/media/C11xS1wUcAAeaKF.jpg",
+        "url": "https://t.co/esnJCdKVol",
+        "display_url": "pic.twitter.com/esnJCdKVol",
+        "expanded_url": "https://twitter.com/pkdev/status/818935308813103104/photo/1",
+        "type": "photo",
+        "sizes": {
+          "medium": {
+            "w": 1200,
+            "h": 800,
+            "resize": "fit"
+          },
+          "small": {
+            "w": 680,
+            "h": 453,
+            "resize": "fit"
+          },
+          "thumb": {
+            "w": 150,
+            "h": 150,
+            "resize": "crop"
+          },
+          "large": {
+            "w": 2048,
+            "h": 1365,
+            "resize": "fit"
+          }
+        }
+      }
+    ]
+  },
+  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
+  "in_reply_to_status_id": null,
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_user_id": null,
+  "in_reply_to_user_id_str": null,
+  "in_reply_to_screen_name": null,
+  "user": {
+    "id": 143883456,
+    "id_str": "143883456",
+    "name": "aaronpk dev",
+    "screen_name": "pkdev",
+    "location": "Portland, OR",
+    "description": "Dev account for testing Twitter things. Follow me here: http://t.co/DtzRLfxayu",
+    "url": "https://t.co/fXLomQaMAd",
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/fXLomQaMAd",
+            "expanded_url": "https://aaronparecki.com/",
+            "display_url": "aaronparecki.com",
+            "indices": [
+              0,
+              23
+            ]
+          }
+        ]
+      },
+      "description": {
+        "urls": [
+          {
+            "url": "http://t.co/DtzRLfxayu",
+            "expanded_url": "https://twitter.com/aaronpk",
+            "display_url": "twitter.com/aaronpk",
+            "indices": [
+              56,
+              78
+            ]
+          }
+        ]
+      }
+    },
+    "protected": true,
+    "followers_count": 4,
+    "friends_count": 1,
+    "listed_count": 1,
+    "created_at": "Fri May 14 17:47:15 +0000 2010",
+    "favourites_count": 1,
+    "utc_offset": -28800,
+    "time_zone": "Pacific Time (US & Canada)",
+    "geo_enabled": true,
+    "verified": false,
+    "statuses_count": 36,
+    "lang": "en",
+    "contributors_enabled": false,
+    "is_translator": false,
+    "is_translation_enabled": false,
+    "profile_background_color": "C0DEED",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_tile": false,
+    "profile_image_url": "http://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_link_color": "1DA1F2",
+    "profile_sidebar_border_color": "C0DEED",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "profile_text_color": "333333",
+    "profile_use_background_image": true,
+    "has_extended_profile": true,
+    "default_profile": true,
+    "default_profile_image": false,
+    "following": true,
+    "follow_request_sent": false,
+    "notifications": false,
+    "translator_type": "none"
+  },
+  "geo": null,
+  "coordinates": null,
+  "place": {
+    "id": "ac88a4f17a51c7fc",
+    "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+    "place_type": "city",
+    "name": "Portland",
+    "full_name": "Portland, OR",
+    "country_code": "US",
+    "country": "United States",
+    "contained_within": [
+
+    ],
+    "bounding_box": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -122.7900653,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.6509405
+          ],
+          [
+            -122.7900653,
+            45.6509405
+          ]
+        ]
+      ]
+    },
+    "attributes": {
+    }
+  },
+  "contributors": null,
+  "is_quote_status": false,
+  "retweet_count": 0,
+  "favorite_count": 0,
+  "favorited": false,
+  "retweeted": false,
+  "possibly_sensitive": false,
+  "possibly_sensitive_appealable": false,
+  "lang": "en"
+}

tests/data/api.twitter.com/818943244553699328.json

@@ -0,0 +1,147 @@
+{
+  "created_at": "Tue Jan 10 22:10:59 +0000 2017",
+  "id": 818943244553699328,
+  "id_str": "818943244553699328",
+  "full_text": "Here 🎉 have an emoji",
+  "truncated": false,
+  "display_text_range": [
+    0,
+    20
+  ],
+  "entities": {
+    "hashtags": [
+
+    ],
+    "symbols": [
+
+    ],
+    "user_mentions": [
+
+    ],
+    "urls": [
+
+    ]
+  },
+  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
+  "in_reply_to_status_id": null,
+  "in_reply_to_status_id_str": null,
+  "in_reply_to_user_id": null,
+  "in_reply_to_user_id_str": null,
+  "in_reply_to_screen_name": null,
+  "user": {
+    "id": 143883456,
+    "id_str": "143883456",
+    "name": "aaronpk dev",
+    "screen_name": "pkdev",
+    "location": "Portland, OR",
+    "description": "Dev account for testing Twitter things. Follow me here: http://t.co/DtzRLfxayu",
+    "url": "https://t.co/fXLomQaMAd",
+    "entities": {
+      "url": {
+        "urls": [
+          {
+            "url": "https://t.co/fXLomQaMAd",
+            "expanded_url": "https://aaronparecki.com/",
+            "display_url": "aaronparecki.com",
+            "indices": [
+              0,
+              23
+            ]
+          }
+        ]
+      },
+      "description": {
+        "urls": [
+          {
+            "url": "http://t.co/DtzRLfxayu",
+            "expanded_url": "https://twitter.com/aaronpk",
+            "display_url": "twitter.com/aaronpk",
+            "indices": [
+              56,
+              78
+            ]
+          }
+        ]
+      }
+    },
+    "protected": true,
+    "followers_count": 4,
+    "friends_count": 1,
+    "listed_count": 1,
+    "created_at": "Fri May 14 17:47:15 +0000 2010",
+    "favourites_count": 1,
+    "utc_offset": -28800,
+    "time_zone": "Pacific Time (US & Canada)",
+    "geo_enabled": true,
+    "verified": false,
+    "statuses_count": 37,
+    "lang": "en",
+    "contributors_enabled": false,
+    "is_translator": false,
+    "is_translation_enabled": false,
+    "profile_background_color": "C0DEED",
+    "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
+    "profile_background_tile": false,
+    "profile_image_url": "http://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_image_url_https": "https://pbs.twimg.com/profile_images/638125135904436224/qd_d94Qn_normal.jpg",
+    "profile_link_color": "1DA1F2",
+    "profile_sidebar_border_color": "C0DEED",
+    "profile_sidebar_fill_color": "DDEEF6",
+    "profile_text_color": "333333",
+    "profile_use_background_image": true,
+    "has_extended_profile": true,
+    "default_profile": true,
+    "default_profile_image": false,
+    "following": true,
+    "follow_request_sent": false,
+    "notifications": false,
+    "translator_type": "none"
+  },
+  "geo": null,
+  "coordinates": null,
+  "place": {
+    "id": "ac88a4f17a51c7fc",
+    "url": "https://api.twitter.com/1.1/geo/id/ac88a4f17a51c7fc.json",
+    "place_type": "city",
+    "name": "Portland",
+    "full_name": "Portland, OR",
+    "country_code": "US",
+    "country": "United States",
+    "contained_within": [
+
+    ],
+    "bounding_box": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -122.7900653,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.421863
+          ],
+          [
+            -122.471751,
+            45.6509405
+          ],
+          [
+            -122.7900653,
+            45.6509405
+          ]
+        ]
+      ]
+    },
+    "attributes": {
+    }
+  },
+  "contributors": null,
+  "is_quote_status": false,
+  "retweet_count": 0,
+  "favorite_count": 0,
+  "favorited": false,
+  "retweeted": false,
+  "lang": "en"
+}