implement backfilling for newly discovered actors

5 files changed, 159 insertions, 23 deletions

diff --git a/Digitigrade/Job/ProcessIncomingSimplePush.php b/Digitigrade/Job/ProcessIncomingSimplePush.php
new file mode 100644
index 0000000..a48a70b
--- /dev/null
+++ b/Digitigrade/Job/ProcessIncomingSimplePush.php
@@ -0,0 +1,44 @@
+<?php
+namespace Digitigrade\Job;
+
+use Digitigrade\Job;
+use Digitigrade\JobQueue;
+use Digitigrade\Logger;
+
+class ProcessIncomingSimplePush extends Job {
+    public string $uri;
+
+    public function __construct(string $uri) {
+        $this->uri = $uri;
+        $this->remainingTries = 3;
+    }
+
+    public function run() {
+        $log = Logger::getInstance();
+
+        $log->info("processing simple push for $this->uri");
+
+        $result = send_request_authenticated($this->uri);
+        if (!str_contains($result->headers[0], '200')) {
+            throw new \RuntimeException("unable to fetch given simple push uri: $this->uri");
+        }
+        $obj = json_decode($result->body);
+
+        if ($obj === null) {
+            throw new \RuntimeException("fetched content doesn't look like valid json");
+        }
+        if (!isset($obj->type, $obj->self)) {
+            throw new \RuntimeException('the object needs to have `type` and `self` properties');
+        }
+        if ($obj->self != $this->uri) {
+            throw new \RuntimeException("fetched object has a different uri from the one provided ($obj->self != $this->uri)");
+        }
+
+        $log->info('fetch successful, pushed object will be dealt with soon');
+        (new ProcessIncomingPush($obj))->submit();
+    }
+
+    public function submitDelayed(int $delaySecs) {
+        JobQueue::getInstance()->submitDelayed($this, $delaySecs);
+    }
+}
\ No newline at end of file
diff --git a/Digitigrade/Model/Actor.php b/Digitigrade/Model/Actor.php
index d7fde42..d18c1f5 100644
--- a/Digitigrade/Model/Actor.php
+++ b/Digitigrade/Model/Actor.php
@@ -2,10 +2,13 @@
 namespace Digitigrade\Model;
 
 use Digitigrade\Db;
+use Digitigrade\Job\ProcessIncomingPush;
+use Digitigrade\Job\ProcessIncomingSimplePush;
 use Digitigrade\Notification\PendingFollowActionedNotif;
 use Digitigrade\Notification\PokeNotif;
 use Digitigrade\Notification\UnblockNotif;
 use Digitigrade\Notification\UnfollowNotif;
+use Digitigrade\Pagination;
 use Digitigrade\PokeVerb;
 use Digitigrade\RpcException;
 use Digitigrade\RpcReceiver;
@@ -90,6 +93,10 @@ class Actor extends PushableModel implements RpcReceiver {
         $this->saveExtensions();
     }
 
+    protected function onNewlyDiscovered() {
+        $this->backfill();
+    }
+
     private function findExtensions() {
         $pdo = Db::getInstance()->getPdo();
         $stmt = $pdo->prepare('SELECT uri, data FROM actor_extension WHERE actor_id = ?');
@@ -139,6 +146,40 @@ class Actor extends PushableModel implements RpcReceiver {
         return self::findByUri($uri);
     }
 
+    /**
+     * Fetches this actor's remote feeds and all objects listed by them.
+     * @param ?int $limit Maximum number of objects to fetch
+     * @return void
+     */
+    public function backfill(?int $limit = null) {
+        if (isset($this->endpoints->fullFeed)) {
+            $this->backfillFromFullFeed($limit);
+        } else {
+            $this->backfillFromBasicFeed($limit);
+        }
+    }
+
+    private function backfillFromFullFeed(?int $limit) {
+        $pagination = Pagination::fromUri($this->endpoints->fullFeed);
+        for ($i = 0; isset($limit) ? ($i < $limit) : true; $i++) {
+            $item = $pagination->next();
+            if ($item === false)
+                break;
+            // pretend they're pushes ... it's good enough
+            (new ProcessIncomingPush($item))->submit();
+        }
+    }
+
+    private function backfillFromBasicFeed(?int $limit) {
+        $pagination = Pagination::fromUri($this->endpoints->basicFeed);
+        for ($i = 0; isset($limit) ? ($i < $limit) : true; $i++) {
+            $uri = $pagination->next();
+            if ($uri === false)
+                break;
+            (new ProcessIncomingSimplePush($uri))->submit();
+        }
+    }
+
     public function rpcCall(string $method, array $args, string $customEndpoint = null) {
         // $args = [Actor $actingAs, ?string $requestBody]
         if (!$this->isLocal) {
diff --git a/Digitigrade/Model/FetchableModel.php b/Digitigrade/Model/FetchableModel.php
index c6bc291..fff7aeb 100644
--- a/Digitigrade/Model/FetchableModel.php
+++ b/Digitigrade/Model/FetchableModel.php
@@ -85,6 +85,14 @@ abstract class FetchableModel extends Model implements RemoteFetchable {
         if (!$obj->validate() || !PolicyManager::getInstance()->check($obj)) {
             return null; // and don't save
         }
+
+        if (isset($obj->uri)) {
+            $isNew = $obj::countWhere('uri = ?', [$obj->uri]) == 0;
+            if ($isNew) {
+                $obj->onNewlyDiscovered();
+            }
+        }
+
         if ($autoSave) {
             $obj->save();
             $obj->finaliseAfterSave();
@@ -99,6 +107,9 @@ abstract class FetchableModel extends Model implements RemoteFetchable {
     protected function finaliseAfterSave() {
     }
 
+    protected function onNewlyDiscovered() {
+    }
+
     public static function findByUri(string $uri, bool $autoSave = true, bool $forceRefetch = false): ?static {
         if (!$forceRefetch) {
             $obj = static::findWhere('uri = ?', [$uri]);
diff --git a/Digitigrade/Pagination.php b/Digitigrade/Pagination.php
new file mode 100644
index 0000000..75a876f
--- /dev/null
+++ b/Digitigrade/Pagination.php
@@ -0,0 +1,60 @@
+<?php
+namespace Digitigrade;
+
+use JsonMapper;
+
+class Pagination {
+    public int $page;
+    public int $totalPages;
+    public int $totalItems;
+    public ?string $nextPage = null;
+    public ?string $previousPage = null;
+    public array $items;
+
+    /**
+     * Fetches a pagination page.
+     * @param string $uri
+     * @param class-string $innerType Type of the contained items.
+     * @return self
+     */
+    public static function fromUri(string $uri): self {
+        $response = send_request_authenticated($uri);
+        $data = json_decode($response->body);
+        if (!isset($data->page)) {
+            throw new \RuntimeException("doesn't look like a pagination: $uri");
+        }
+        $p = (new JsonMapper())->map($data, self::class);
+        return $p;
+    }
+
+    private function become(self $other) {
+        $this->page = $other->page;
+        $this->totalPages = $other->totalPages;
+        $this->totalItems = $other->totalItems;
+        $this->nextPage = $other->nextPage;
+        $this->previousPage = $other->previousPage;
+        $this->items = &$other->items;
+    }
+
+    /**
+     * Gets the next item from the pagination, going to the next page if needed.
+     * 
+     * Careful! This behaves differently from PHP's built in `next`: it returns
+     * the first item, the first time it's called, rather than the second.
+     * @return mixed The next item, or `false` if at the end.
+     */
+    public function next(): mixed {
+        if (current($this->items) === false) {
+            if (isset($this->nextPage)) {
+                $this->become(self::fromUri($this->nextPage));
+                return $this->next();
+            } else {
+                return false;
+            }
+        } else {
+            $item = current($this->items);
+            next($this->items);
+            return $item;
+        }
+    }
+}
\ No newline at end of file
diff --git a/routes/push.php b/routes/push.php
index 344412c..2420f67 100644
--- a/routes/push.php
+++ b/routes/push.php
@@ -4,6 +4,7 @@ use Digitigrade\HttpResponseStatus\BadRequest;
 use Digitigrade\HttpResponseStatus\Forbidden;
 use Digitigrade\HttpResponseStatus\Unauthorized;
 use Digitigrade\Job\ProcessIncomingPush;
+use Digitigrade\Job\ProcessIncomingSimplePush;
 use Digitigrade\Model\Instance;
 use Digitigrade\Router;
 
@@ -45,32 +46,11 @@ Router::getInstance()->mount('/push/simple', function (array $args) {
         throw new BadRequest('dodgy looking uri!');
     }
 
-    $result = send_request_authenticated($uri);
-    if (!str_contains($result->headers[0], '200')) {
-        throw new BadRequest('unable to fetch that uri!');
-    }
-    $obj = json_decode($result->body);
-
-    if ($obj === null) {
-        throw new BadRequest("fetched content doesn't look like valid json");
-    }
-    if (!isset($obj->type, $obj->self)) {
-        throw new BadRequest('the object needs to have `type` and `self` properties');
-    }
-    if ($obj->self != $uri) {
-        throw new BadRequest('fetched object has a different uri from the one you gave me');
-    }
-    if (!in_array($obj->type, ['actor', 'note', 'interaction', 'extension', 'tombstone'])) {
-        throw new BadRequest('invalid object type!');
-    }
-
-    // only add ratelimiting delay after having validated it, to mitigate the
-    // possibility of denial-of-service attacks (still possible though)
     // FIXME: use a proper rate limit system
     sleep(5);
     // and only add the job if the client actually waited for the delay time
     if (!connection_aborted()) {
-        // also wait a minute before processing the push
-        (new ProcessIncomingPush($obj))->submitDelayed(60);
+        // submit it for later, to reduce abuse, theoretically
+        (new ProcessIncomingSimplePush($uri))->submitDelayed(60);
     }
 });
\ No newline at end of file