parse basic ActivityStreams objects

including from rel=alternate

lib/XRay/Fetcher.php

@@ -71,7 +71,7 @@ class Fetcher {
 
     $headers = [];
 
-    $headers[] = 'Accept: text/html, application/json, application/xml, text/xml';
+    $headers[] = 'Accept: application/mf2+json, application/activity+json, text/html, application/json, application/xml, text/xml';
 
     if(isset($opts['token']))
       $headers[] = 'Authorization: Bearer ' . $opts['token'];

lib/XRay/Formats/ActivityStreams.php

@@ -0,0 +1,179 @@
+<?php
+namespace p3k\XRay\Formats;
+
+use DateTime;
+use \p3k\XRay\PostType;
+
+class ActivityStreams extends Format {
+
+  public static function is_as2_json($document) {
+    if(is_array($document) && isset($document['@context'])) {
+      if(is_string($document['@context']) && $document['@context'] == 'https://www.w3.org/ns/activitystreams')
+        return true;
+      if(is_array($document['@context']) && in_array('https://www.w3.org/ns/activitystreams', $document['@context']))
+        return true;
+    }
+
+    return false;
+  }
+
+  public static function matches_host($url) {
+    return true;
+  }
+
+  public static function matches($url) {
+    return true;
+  }
+
+  public static function parse($as2, $url, $http, $opts=[]) {
+    if(!isset($as2['type']))
+      return false;
+
+    switch($as2['type']) {
+      case 'Person':
+        return self::parseAsHCard($as2, $url, $http, $opts);
+      case 'Note':
+        return self::parseAsHEntry($as2, $url, $http, $opts);
+    }
+
+    $result = [
+      'data' => [
+        'type' => 'unknown',
+      ],
+      'url' => $url,
+    ];
+    return $result;
+  }
+
+  private static function parseAsHEntry($as2, $url, $http, $opts) {
+    $data = [
+      'type' => 'entry'
+    ];
+    $refs = [];
+
+    if(isset($as2['url']))
+      $data['url'] = $as2['url'];
+    elseif(isset($as2['id']))
+      $data['url'] = $as2['id'];
+
+    if(isset($as2['published'])) {
+      try {
+        $date = new DateTime($as2['published']);
+        $data['published'] = $date->format('c');
+      } catch(\Exception $e){}
+    }
+
+    if(isset($as2['content'])) {
+      $html = trim(self::sanitizeHTML($as2['content']));
+      $text = trim(self::stripHTML($html));
+
+      $data['content'] = [
+        'text' => $text
+      ];
+
+      if($html && $text && $text != $html) {
+        $data['content']['html'] = $html;
+      }
+    }
+
+    if(isset($as2['tag']) && is_array($as2['tag'])) {
+      $emoji = [];
+      $category = [];
+      foreach($as2['tag'] as $tag) {
+        if(is_array($tag) && isset($tag['name']) && isset($tag['type']) && $tag['type'] == 'Hashtag')
+          $category[] = trim($tag['name'], '#');
+        if(is_array($tag) && isset($tag['type']) && $tag['type'] == 'Emoji' && isset($tag['icon']['url'])) {
+          $emoji[$tag['name']] = $tag['icon']['url'];
+        }
+      }
+
+      if(count($category))
+        $data['category'] = $category;
+
+      if(count($emoji) && isset($data['content']['html'])) {
+        foreach($emoji as $code=>$img) {
+          $data['content']['html'] = str_replace($code, '<img src="'.$img.'" alt="'.$code.'" title="'.$code.'" height="24" class="xray-custom-emoji">', $data['content']['html']);
+        }
+      }
+    }
+
+    if(isset($as2['inReplyTo'])) {
+      $data['in-reply-to'] = [$as2['inReplyTo']];
+    }
+
+    // Photos and Videos
+    if(isset($as2['attachment'])) {
+      $photos = [];
+      $videos = [];
+      foreach($as2['attachment'] as $attachment) {
+        if(strpos($attachment['mediaType'], 'image/') !== false) {
+          $photos[] = $attachment['url'];
+        }
+        if(strpos($attachment['mediaType'], 'video/') !== false) {
+          $videos[] = $attachment['url'];
+        }
+      }
+      if(count($photos))
+        $data['photo'] = $photos;
+      if(count($videos))
+        $data['video'] = $videos;
+    }
+
+    // Fetch the author info, which requires an HTTP request
+    if(isset($as2['attributedTo']) && is_string($as2['attributedTo'])) {
+      $authorResponse = $http->get($as2['attributedTo'], ['Accept: application/activity+json,application/json']);
+      if($authorResponse && !empty($authorResponse['body'])) {
+        $authorProfile = json_decode($authorResponse['body'], true);
+        $author = self::parseAsHCard($authorProfile, $as2['attributedTo'], $http, $opts);
+        if($author && !empty($author['data']))
+          $data['author'] = $author['data'];
+      }
+    }
+
+    $data['post-type'] = PostType::discover($data);
+
+    $response = [
+      'data' => $data,
+    ];
+
+    if(count($refs)) {
+      $response['data']['refs'] = $refs;
+    }
+
+    return $response;
+  }
+
+  private static function parseAsHCard($as2, $url, $http, $opts) {
+    $data = [
+      'type' => 'card',
+      'name' => null,
+      'url' => null,
+      'photo' => null
+    ];
+
+    if(!empty($as2['name']))
+      $data['name'] = $as2['name'];
+    elseif(isset($as2['preferredUsername']))
+      $data['name'] = $as2['preferredUsername'];
+
+    if(isset($as2['preferredUsername']))
+      $data['nickname'] = $as2['preferredUsername'];
+
+    if(isset($as2['url']))
+      $data['url'] = $as2['url'];
+
+    if(isset($as2['icon']) && isset($as2['icon']['url']))
+      $data['photo'] = $as2['icon']['url'];
+
+    // TODO: featured image for h-cards?
+    // if(isset($as2['image']) && isset($as2['image']['url']))
+    //   $data['featured'] = $as2['image']['url'];
+
+    $response = [
+      'data' => $data
+    ];
+
+    return $response;
+  }
+
+}

lib/XRay/Formats/Format.php

@@ -59,7 +59,8 @@ abstract class Format implements iFormat {
       'h6',
       'ul',
       'li',
-      'ol'
+      'ol',
+      'span',
     ];
     if($allowImg)
       $allowed[] = 'img';

lib/XRay/Formats/HTML.php

@@ -95,26 +95,63 @@ class HTML extends Format {
 
     // Check for a rel=alternate link to a Microformats JSON representation, and use that instead
     if(isset($mf2['rel-urls'])) {
+      $alternates = [
+        'mf2' => [],
+        'as2' => [],
+      ];
+
       foreach($mf2['rel-urls'] as $relurl => $reltype) {
+
         if(in_array('alternate', $reltype['rels']) && $reltype['type'] == 'application/mf2+json') {
-          // Fetch and parse the MF2 JSON link instead
-          $jsonpage = $http->get($relurl, [
-            'Accept' => 'application/mf2+json,application/json'
-          ]);
-          // Skip and fall back to parsing the HTML if anything about this request fails
-          if(!$jsonpage['error'] && $jsonpage['body']) {
-            $jsondata = json_decode($jsonpage['body'],true);
-            if($jsondata) {
-              $data = Formats\Mf2::parse($jsondata, $url, $http, $opts);
-              if($data && is_array($data) && isset($data['data']['type'])) {
-                $data['url'] = $relurl;
-                $data['source-format'] = 'mf2+json';
-                return $data;
-              }
+          $alternates['mf2'][] = $relurl;
+        }
+
+        if(in_array('alternate', $reltype['rels']) && $reltype['type'] == 'application/activity+json') {
+          $alternates['as2'][] = $relurl;
+        }
+
+      }
+
+      if(count($alternates['mf2'])) {
+        // Fetch and parse the MF2 JSON link
+        $relurl = $alternates['mf2'][0];
+        $jsonpage = $http->get($relurl, [
+          'Accept' => 'application/mf2+json,application/json'
+        ]);
+        // Skip and fall back to parsing the HTML if anything about this request fails
+        if(!$jsonpage['error'] && $jsonpage['body']) {
+          $jsondata = json_decode($jsonpage['body'],true);
+          if($jsondata) {
+            $data = Formats\Mf2::parse($jsondata, $url, $http, $opts);
+            if($data && is_array($data) && isset($data['data']['type'])) {
+              $data['url'] = $relurl;
+              $data['source-format'] = 'mf2+json';
+              return $data;
             }
           }
         }
       }
+
+      if(count($alternates['as2'])) {
+        $relurl = $alternates['as2'][0];
+        // Fetch and parse the ActivityStreams JSON link
+        $jsonpage = $http->get($relurl, [
+          'Accept' => 'application/activity+json,application/json'
+        ]);
+        // Skip and fall back to parsing the HTML if anything about this request fails
+        if(!$jsonpage['error'] && $jsonpage['body']) {
+          $jsondata = json_decode($jsonpage['body'],true);
+          if($jsondata) {
+            $data = Formats\ActivityStreams::parse($jsondata, $url, $http, $opts);
+            if($data && is_array($data) && isset($data['data']['type'])) {
+              $data['url'] = $relurl;
+              $data['source-format'] = 'activity+json';
+              return $data;
+            }
+          }
+        }
+      }
+
     }
 
     // Now start pulling in the data from the page. Start by looking for microformats2

lib/XRay/Formats/Mf2.php

@@ -1,8 +1,6 @@
 <?php
 namespace p3k\XRay\Formats;
 
-use HTMLPurifier, HTMLPurifier_Config;
-
 class Mf2 extends Format {
 
   use Mf2Feed;

lib/XRay/Parser.php

@@ -49,6 +49,13 @@ class Parser {
       return $data;
     }
 
+    // Check if an ActivityStreams JSON object was passed in
+    if(Formats\ActivityStreams::is_as2_json($body)) {
+      $data = Formats\ActivityStreams::parse($body, $url, $this->http, $opts);
+      $data['source-format'] = 'activity+json';
+      return $data;
+    }
+
     if(substr($body, 0, 5) == '<?xml') {
       return Formats\XML::parse($body, $url);
     }
@@ -57,18 +64,22 @@ class Parser {
       $parsed = json_decode($body, true);
       if($parsed && isset($parsed['version']) && $parsed['version'] == 'https://jsonfeed.org/version/1') {
         return Formats\JSONFeed::parse($parsed, $url);
-      // TODO: check for an activitystreams object too
       } elseif($parsed && isset($parsed['items'][0]['type']) && isset($parsed['items'][0]['properties'])) {
         // Check if an mf2 JSON string was passed in
         $data = Formats\Mf2::parse($parsed, $url, $this->http, $opts);
         $data['source-format'] = 'mf2+json';
         return $data;
+      } elseif($parsed && Formats\ActivityStreams::is_as2_json($parsed)) {
+        // Check if an ActivityStreams JSON string was passed in
+        $data = Formats\ActivityStreams::parse($parsed, $url, $this->http, $opts);
+        $data['source-format'] = 'activity+json';
+        return $data;
       }
     }
 
     // No special parsers matched, parse for Microformats now
     $data = Formats\HTML::parse($this->http, $body, $url, $opts);
-    if(!isset($data['source-format']))
+    if(!isset($data['source-format']) && isset($data['type']) && $data['type'] != 'unknown')
       $data['source-format'] = 'mf2+html';
     return $data;
   }

tests/ActivityStreamsTest.php

@@ -0,0 +1,142 @@
+<?php
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+
+class ActivityStreamsTest extends PHPUnit_Framework_TestCase {
+
+  private $http;
+
+  public function setUp() {
+    $this->client = new Parse();
+    $this->client->http = new p3k\HTTP\Test(dirname(__FILE__).'/data/');
+    $this->client->mc = null;
+  }
+
+  private function parse($params) {
+    $request = new Request($params);
+    $response = new Response();
+    return $this->client->parse($request, $response);
+  }
+
+  public function testAuthorProfile() {
+    $url = 'http://activitystreams.example/aaronpk';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('activity+json', $data['source-format']);
+    $this->assertEquals('card', $data['data']['type']);
+    $this->assertEquals('aaronpk', $data['data']['name']);
+    $this->assertEquals('https://aaronparecki.com/images/profile.jpg', $data['data']['photo']);
+    $this->assertEquals('https://aaronparecki.com/', $data['data']['url']);
+  }
+
+  public function testNoteWithTags() {
+    $url = 'http://activitystreams.example/note.json';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('activity+json', $data['source-format']);
+    $this->assertEquals('note', $data['data']['post-type']);
+    $this->assertEquals($url, $data['data']['url']);
+    $this->assertEquals('2018-07-12T13:02:04-07:00', $data['data']['published']);
+    $this->assertEquals('This is the text content of an ActivityStreams note', $data['data']['content']['text']);
+    $this->assertArrayNotHasKey('html', $data['data']['content']);
+    $this->assertSame(['activitystreams'], $data['data']['category']);
+    $this->assertEquals('aaronpk', $data['data']['author']['name']);
+    $this->assertEquals('https://aaronparecki.com/images/profile.jpg', $data['data']['author']['photo']);
+    $this->assertEquals('https://aaronparecki.com/', $data['data']['author']['url']);
+  }
+
+  public function testPhoto() {
+    $url = 'http://activitystreams.example/photo.json';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('activity+json', $data['source-format']);
+    $this->assertEquals($url, $data['data']['url']);
+    $this->assertEquals('photo', $data['data']['post-type']);
+    $this->assertEquals('2018-07-12T13:02:04-07:00', $data['data']['published']);
+    $this->assertEquals('This is the text content of an ActivityStreams photo', $data['data']['content']['text']);
+    $this->assertArrayNotHasKey('html', $data['data']['content']);
+    $this->assertSame(['activitystreams'], $data['data']['category']);
+    $this->assertSame(['https://aaronparecki.com/2018/06/28/26/photo.jpg'], $data['data']['photo']);
+  }
+
+  public function testVideo() {
+    $url = 'http://activitystreams.example/video.json';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('activity+json', $data['source-format']);
+    $this->assertEquals('video', $data['data']['post-type']);
+    $this->assertEquals('2018-07-12T13:02:04-07:00', $data['data']['published']);
+    $this->assertSame(['https://aaronparecki.com/2018/07/21/19/video.mp4'], $data['data']['video']);
+  }
+
+  public function testReply() {
+    $url = 'http://activitystreams.example/reply.json';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('activity+json', $data['source-format']);
+    $this->assertEquals('reply', $data['data']['post-type']);
+    $this->assertEquals('2018-07-12T13:02:04-07:00', $data['data']['published']);
+    $this->assertArrayNotHasKey('category', $data['data']); // should not include the person-tag
+    // For now, don't fetch the reply context
+    $this->assertEquals(['http://activitystreams.example/note.json'], $data['data']['in-reply-to']);
+  }
+
+  public function testCustomEmoji() {
+    $url = 'http://activitystreams.example/custom-emoji.json';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('activity+json', $data['source-format']);
+    $this->assertEquals('note', $data['data']['post-type']);
+    $this->assertEquals("https://mastodon.social/@Gargron/100465999501820229", $data['data']['url']);
+    $this->assertEquals('2018-07-30T22:24:54+00:00', $data['data']['published']);
+    $this->assertEquals(':yikes:', $data['data']['content']['text']);
+    $this->assertEquals('<p><img src="https://files.mastodon.social/custom_emojis/images/000/031/275/original/yikes.png" alt=":yikes:" title=":yikes:" height="24" class="xray-custom-emoji"></p>', $data['data']['content']['html']);
+    $this->assertEquals('Eugen', $data['data']['author']['name']);
+    $this->assertEquals('Gargron', $data['data']['author']['nickname']);
+    $this->assertEquals('https://files.mastodon.social/accounts/avatars/000/000/001/original/eb9e00274b135808.png', $data['data']['author']['photo']);
+    $this->assertEquals('https://mastodon.social/@Gargron', $data['data']['author']['url']);
+  }
+
+  public function testRelAlternatePriority() {
+    $url = 'http://source.example.com/rel-alternate-as2';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('activity+json', $data['source-format']);
+    $this->assertEquals('http://activitystreams.example/note.json', $data['parsed-url']);
+    $this->assertEquals('http://source.example.com/rel-alternate-as2', $data['url']);
+    $this->assertEquals('note', $data['data']['post-type']);
+    $this->assertEquals('2018-07-12T13:02:04-07:00', $data['data']['published']);
+    $this->assertEquals('This is the text content of an ActivityStreams note', $data['data']['content']['text']);
+    $this->assertArrayNotHasKey('html', $data['data']['content']);
+    $this->assertSame(['activitystreams'], $data['data']['category']);
+  }
+
+}

tests/ParseTest.php

@@ -915,6 +915,19 @@ class ParseTest extends PHPUnit_Framework_TestCase {
     $this->assertEquals('This should be the content from XRay', $data['data']['content']['text']);
   }
 
+  public function testRelAlternatePrioritizesMf2OverAS2() {
+    $url = 'http://source.example.com/rel-alternate-priority-mf2-as2';
+    $response = $this->parse(['url' => $url]);
+
+    $body = $response->getContent();
+    $this->assertEquals(200, $response->getStatusCode());
+    $data = json_decode($body, true);
+
+    $this->assertEquals('mf2+json', $data['source-format']);
+    $this->assertEquals('http://source.example.com/rel-alternate-priority.json', $data['parsed-url']);
+    $this->assertEquals('This should be the content from XRay', $data['data']['content']['text']);
+  }
+
   public function testRelAlternateFallsBackOnInvalidJSON() {
     $url = 'http://source.example.com/rel-alternate-fallback';
     $response = $this->parse(['url' => $url]);

tests/data/activitystreams.example/Gargron

@@ -0,0 +1,7 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 30 Jul 2018 03:29:14 GMT
+Content-Type: application/activity+json
+Connection: keep-alive
+
+{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","movedTo":{"@id":"as:movedTo","@type":"@id"},"Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji","focalPoint":{"@container":"@list","@id":"toot:focalPoint"},"featured":{"@id":"toot:featured","@type":"@id"},"schema":"http://schema.org#","PropertyValue":"schema:PropertyValue","value":"schema:value"}],"id":"https://mastodon.social/users/Gargron","type":"Person","following":"https://mastodon.social/users/Gargron/following","followers":"https://mastodon.social/users/Gargron/followers","inbox":"https://mastodon.social/users/Gargron/inbox","outbox":"https://mastodon.social/users/Gargron/outbox","featured":"https://mastodon.social/users/Gargron/collections/featured","preferredUsername":"Gargron","name":"Eugen","summary":"\u003cp\u003eDeveloper of Mastodon. 25\u003c/p\u003e","url":"https://mastodon.social/@Gargron","manuallyApprovesFollowers":false,"publicKey":{"id":"https://mastodon.social/users/Gargron#main-key","owner":"https://mastodon.social/users/Gargron","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvXc4vkECU2/CeuSo1wtn\nFoim94Ne1jBMYxTZ9wm2YTdJq1oiZKif06I2fOqDzY/4q/S9uccrE9Bkajv1dnkO\nVm31QjWlhVpSKynVxEWjVBO5Ienue8gND0xvHIuXf87o61poqjEoepvsQFElA5ym\novljWGSA/jpj7ozygUZhCXtaS2W5AD5tnBQUpcO0lhItYPYTjnmzcc4y2NbJV8hz\n2s2G8qKv8fyimE23gY1XrPJg+cRF+g4PqFXujjlJ7MihD9oqtLGxbu7o1cifTn3x\nBfIdPythWu5b4cujNsB3m3awJjVmx+MHQ9SugkSIYXV0Ina77cTNS0M2PYiH1PFR\nTwIDAQAB\n-----END PUBLIC KEY-----\n"},"tag":[],"attachment":[{"type":"PropertyValue","name":"Patreon","value":"\u003ca href=\"https://www.patreon.com/mastodon\" rel=\"me nofollow noopener\" target=\"_blank\"\u003e\u003cspan class=\"invisible\"\u003ehttps://www.\u003c/span\u003e\u003cspan class=\"\"\u003epatreon.com/mastodon\u003c/span\u003e\u003cspan class=\"invisible\"\u003e\u003c/span\u003e\u003c/a\u003e"},{"type":"PropertyValue","name":"E-mail","value":"eugen@zeonfederated.com"}],"endpoints":{"sharedInbox":"https://mastodon.social/inbox"},"icon":{"type":"Image","mediaType":"image/png","url":"https://files.mastodon.social/accounts/avatars/000/000/001/original/eb9e00274b135808.png"},"image":{"type":"Image","mediaType":"image/jpeg","url":"https://files.mastodon.social/accounts/headers/000/000/001/original/998815725e9554b0.jpg"}}

tests/data/activitystreams.example/aaronpk

@@ -0,0 +1,33 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 30 Jul 2018 03:29:14 GMT
+Content-Type: application/activity+json
+Connection: keep-alive
+
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://w3id.org/security/v1"
+  ],
+  "id": "https://aaronparecki.com/aaronpk",
+  "type": "Person",
+  "preferredUsername": "aaronpk",
+  "url": "https://aaronparecki.com/",
+  "icon": {
+    "type": "Image",
+    "mediaType": "image/jpeg",
+    "url": "https://aaronparecki.com/images/profile.jpg"
+  },
+  "image": {
+    "type": "Image",
+    "mediaType": "image/jpeg",
+    "url": "https://aaronparecki.com/images/cover-photo.jpg"
+  },
+  "inbox": "https://aaronparecki.com/activitypub/inbox",
+  "outbox": "https://aaronparecki.com/activitypub/outbox",
+  "publicKey": {
+    "id": "https://aaronparecki.com/aaronpk#key",
+    "owner": "https://aaronparecki.com/aaronpk",
+    "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzlRsHgaSwaoB/AOLxjsU\nBDGDZJileQyg2ZmEhWBMLaqNMJH0EqnraQRHfnHz7Yp32zjYCj2NObaX7ySd3uUJ\nzoEyhmfx122WQupOPclGYBC7mbXXhiheDb3ItPn9TxCOveC8d5d2Bm6vyCq4m31x\nBH0jOImL3AscLNwhEdYIHvweXuIqaat50O6yrgJUadJBvw0hyPVFvwiak1dKA2Su\nHCxsgLpxasEoByJMNy1COG8AvR+SuSvwJXZ2DeDS98Ji9EbeaKl6F2mJGJC/Fe2q\nz0t3mVll8Zs4bdnraw6pcnmFNzOv0SziunJoKjQ5IgZDmWHQY4EgEybxa9SEOZ/s\ntQIDAQAB\n-----END PUBLIC KEY-----\n"
+  }
+}

tests/data/activitystreams.example/custom-emoji.json

@@ -0,0 +1,7 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 30 Jul 2018 03:29:14 GMT
+Content-Type: application/activity+json
+Connection: keep-alive
+
+{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","movedTo":{"@id":"as:movedTo","@type":"@id"},"Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji","focalPoint":{"@container":"@list","@id":"toot:focalPoint"},"featured":{"@id":"toot:featured","@type":"@id"},"schema":"http://schema.org#","PropertyValue":"schema:PropertyValue","value":"schema:value"}],"id":"https://mastodon.social/users/Gargron/statuses/100465999501820229","type":"Note","summary":null,"inReplyTo":null,"published":"2018-07-30T22:24:54Z","url":"https://mastodon.social/@Gargron/100465999501820229","attributedTo":"https://activitystreams.example/Gargron","to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mastodon.social/users/Gargron/followers"],"sensitive":false,"atomUri":"https://mastodon.social/users/Gargron/statuses/100465999501820229","inReplyToAtomUri":null,"conversation":"tag:mastodon.social,2018-07-30:objectId=44299858:objectType=Conversation","content":"\u003cp\u003e:yikes:\u003c/p\u003e","contentMap":{"en":"\u003cp\u003e:yikes:\u003c/p\u003e"},"attachment":[],"tag":[{"id":"https://mastodon.social/emojis/31275","type":"Emoji","name":":yikes:","updated":"2018-07-15T17:28:20Z","icon":{"type":"Image","mediaType":"image/png","url":"https://files.mastodon.social/custom_emojis/images/000/031/275/original/yikes.png"}}]}

tests/data/activitystreams.example/note.json

@@ -0,0 +1,24 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 30 Jul 2018 03:29:14 GMT
+Content-Type: application/activity+json
+Connection: keep-alive
+
+{
+    "@context": "https://www.w3.org/ns/activitystreams",
+    "id": "http://activitystreams.example/note.json",
+    "type": "Note",
+    "published": "2018-07-12T13:02:04-07:00",
+    "attributedTo": "https://activitystreams.example/aaronpk",
+    "content": "This is the text content of an ActivityStreams note",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "tag": [
+        {
+            "id": "https://aaronparecki.com/tag/activitystreams",
+            "name": "#activitystreams",
+            "type": "Hashtag"
+        }
+    ]
+}

tests/data/activitystreams.example/photo.json

@@ -0,0 +1,32 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 30 Jul 2018 03:29:14 GMT
+Content-Type: application/activity+json
+Connection: keep-alive
+
+{
+    "@context": "https://www.w3.org/ns/activitystreams",
+    "id": "http://activitystreams.example/photo.json",
+    "type": "Note",
+    "published": "2018-07-12T13:02:04-07:00",
+    "attributedTo": "https://activitystreams.example/aaronpk",
+    "content": "This is the text content of an ActivityStreams photo",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "tag": [
+        {
+            "type": "Hashtag",
+            "id": "https://aaronparecki.com/tag/activitystreams",
+            "name": "#activitystreams"
+        }
+    ],
+    "attachment": [
+        {
+            "type": "Image",
+            "mediaType": "image/jpeg",
+            "url": "https://aaronparecki.com/2018/06/28/26/photo.jpg",
+            "name": null
+        }
+    ]
+}

tests/data/activitystreams.example/reply.json

@@ -0,0 +1,25 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 30 Jul 2018 03:29:14 GMT
+Content-Type: application/activity+json
+Connection: keep-alive
+
+{
+    "@context": "https://www.w3.org/ns/activitystreams",
+    "id": "http://activitystreams.example/reply.json",
+    "type": "Note",
+    "published": "2018-07-12T13:02:04-07:00",
+    "attributedTo": "https://activitystreams.example/aaronpk",
+    "content": "<a href=\"http://activitystreams.example/aaronpk\">@aaronpk</a> This is a reply",
+    "inReplyTo": "http://activitystreams.example/note.json",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "tag": [
+      {
+        "type": "Mention",
+        "href": "http://activitystreams.example/aaronpk",
+        "name": "@aaronpk@activitystreams.example"
+      }
+    ]
+}

tests/data/activitystreams.example/video.json

@@ -0,0 +1,31 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 30 Jul 2018 03:29:14 GMT
+Content-Type: application/activity+json
+Connection: keep-alive
+
+{
+    "@context": "https://www.w3.org/ns/activitystreams",
+    "id": "http://activitystreams.example/video.json",
+    "type": "Note",
+    "published": "2018-07-12T13:02:04-07:00",
+    "attributedTo": "https://activitystreams.example/aaronpk",
+    "content": "This is the text content of an ActivityStreams photo",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "tag": [
+        {
+            "id": "https://aaronparecki.com/tag/activitystreams",
+            "name": "#activitystreams"
+        }
+    ],
+    "attachment": [
+        {
+            "type": "Document",
+            "mediaType": "video/mp4",
+            "url": "https://aaronparecki.com/2018/07/21/19/video.mp4",
+            "name": null
+        }
+    ]
+}

tests/data/source.example.com/rel-alternate-as2

@@ -0,0 +1,19 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 09 Dec 2015 03:29:14 GMT
+Content-Type: text/html
+Connection: keep-alive
+
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title>Test</title>
+    <link rel="alternate" type="application/activity+json" href="http://activitystreams.example/note.json" />
+  </head>
+  <body>
+    <div class="h-entry">
+      <p class="p-content">This should not be the content from XRay</p>
+    </div>
+  </body>
+</html>

tests/data/source.example.com/rel-alternate-priority-mf2-as2

@@ -0,0 +1,20 @@
+HTTP/1.1 200 OK
+Server: Apache
+Date: Wed, 09 Dec 2015 03:29:14 GMT
+Content-Type: text/html
+Connection: keep-alive
+
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title>Test</title>
+    <link rel="alternate" type="application/activity+json" href="http://activitystreams.example/note.json" />
+    <link rel="alternate" type="application/mf2+json" href="http://source.example.com/rel-alternate-priority.json" />
+  </head>
+  <body>
+    <div class="h-entry">
+      <p class="p-content">This should not be the content from XRay</p>
+    </div>
+  </body>
+</html>