From 5d29d0cc7bb47cad7a662f1e6afc10cb97fe3ddd Mon Sep 17 00:00:00 2001
From: Dan Brown <ssddanbrown@googlemail.com>
Date: Wed, 17 Aug 2022 14:39:53 +0100
Subject: [PATCH] Added reference storage system, and command to re-index

Also re-named/orgranized some files for this, to make them "References"
specific instead of a subset of "Util".
---
 .../Commands/RegenerateCommentContent.php     |  8 ++-
 .../Commands/RegeneratePermissions.php        |  1 +
 app/Console/Commands/RegenerateReferences.php | 58 +++++++++++++++
 app/Entities/Models/Entity.php                | 17 +++++
 .../CrossLinkParser.php                       | 16 ++---
 .../ModelResolvers/BookLinkModelResolver.php  |  4 +-
 .../BookshelfLinkModelResolver.php            |  4 +-
 .../ChapterLinkModelResolver.php              |  4 +-
 .../ModelResolvers/CrossLinkModelResolver.php |  2 +-
 .../ModelResolvers/PageLinkModelResolver.php  |  4 +-
 .../PagePermalinkModelResolver.php            |  2 +-
 app/References/Reference.php                  | 26 +++++++
 app/References/ReferenceService.php           | 71 +++++++++++++++++++
 ...2_08_17_092941_create_references_table.php | 34 +++++++++
 .../CrossLinkParserTest.php                   | 25 ++++++-
 15 files changed, 253 insertions(+), 23 deletions(-)
 create mode 100644 app/Console/Commands/RegenerateReferences.php
 rename app/{Util/CrossLinking => References}/CrossLinkParser.php (82%)
 rename app/{Util/CrossLinking => References}/ModelResolvers/BookLinkModelResolver.php (87%)
 rename app/{Util/CrossLinking => References}/ModelResolvers/BookshelfLinkModelResolver.php (88%)
 rename app/{Util/CrossLinking => References}/ModelResolvers/ChapterLinkModelResolver.php (85%)
 rename app/{Util/CrossLinking => References}/ModelResolvers/CrossLinkModelResolver.php (77%)
 rename app/{Util/CrossLinking => References}/ModelResolvers/PageLinkModelResolver.php (85%)
 rename app/{Util/CrossLinking => References}/ModelResolvers/PagePermalinkModelResolver.php (90%)
 create mode 100644 app/References/Reference.php
 create mode 100644 app/References/ReferenceService.php
 create mode 100644 database/migrations/2022_08_17_092941_create_references_table.php
 rename tests/{Util => References}/CrossLinkParserTest.php (67%)

diff --git a/app/Console/Commands/RegenerateCommentContent.php b/app/Console/Commands/RegenerateCommentContent.php
index 587a5edb3..9da48fb0e 100644
--- a/app/Console/Commands/RegenerateCommentContent.php
+++ b/app/Console/Commands/RegenerateCommentContent.php
@@ -5,6 +5,7 @@ namespace BookStack\Console\Commands;
 use BookStack\Actions\Comment;
 use BookStack\Actions\CommentRepo;
 use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
 
 class RegenerateCommentContent extends Command
 {
@@ -43,9 +44,9 @@ class RegenerateCommentContent extends Command
      */
     public function handle()
     {
-        $connection = \DB::getDefaultConnection();
+        $connection = DB::getDefaultConnection();
         if ($this->option('database') !== null) {
-            \DB::setDefaultConnection($this->option('database'));
+            DB::setDefaultConnection($this->option('database'));
         }
 
         Comment::query()->chunk(100, function ($comments) {
@@ -55,7 +56,8 @@ class RegenerateCommentContent extends Command
             }
         });
 
-        \DB::setDefaultConnection($connection);
+        DB::setDefaultConnection($connection);
         $this->comment('Comment HTML content has been regenerated');
+        return 0;
     }
 }
diff --git a/app/Console/Commands/RegeneratePermissions.php b/app/Console/Commands/RegeneratePermissions.php
index 3396a445f..74f96fd42 100644
--- a/app/Console/Commands/RegeneratePermissions.php
+++ b/app/Console/Commands/RegeneratePermissions.php
@@ -50,5 +50,6 @@ class RegeneratePermissions extends Command
 
         DB::setDefaultConnection($connection);
         $this->comment('Permissions regenerated');
+        return 0;
     }
 }
diff --git a/app/Console/Commands/RegenerateReferences.php b/app/Console/Commands/RegenerateReferences.php
new file mode 100644
index 000000000..93450c5ea
--- /dev/null
+++ b/app/Console/Commands/RegenerateReferences.php
@@ -0,0 +1,58 @@
+<?php
+
+namespace BookStack\Console\Commands;
+
+use BookStack\References\ReferenceService;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class RegenerateReferences extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'bookstack:regenerate-references {--database= : The database connection to use.}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Regenerate all the cross-item model reference index';
+
+    protected ReferenceService $references;
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct(ReferenceService $references)
+    {
+        $this->references = $references;
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     */
+    public function handle()
+    {
+        $connection = DB::getDefaultConnection();
+
+        if ($this->option('database')) {
+            DB::setDefaultConnection($this->option('database'));
+        }
+
+        $this->references->updateForAllPages();
+
+        DB::setDefaultConnection($connection);
+
+        $this->comment('References have been regenerated');
+        return 0;
+    }
+}
diff --git a/app/Entities/Models/Entity.php b/app/Entities/Models/Entity.php
index ffb9b9c7d..26a52073e 100644
--- a/app/Entities/Models/Entity.php
+++ b/app/Entities/Models/Entity.php
@@ -18,6 +18,7 @@ use BookStack\Interfaces\Loggable;
 use BookStack\Interfaces\Sluggable;
 use BookStack\Interfaces\Viewable;
 use BookStack\Model;
+use BookStack\References\Reference;
 use BookStack\Search\SearchIndex;
 use BookStack\Search\SearchTerm;
 use BookStack\Traits\HasCreatorAndUpdater;
@@ -203,6 +204,22 @@ abstract class Entity extends Model implements Sluggable, Favouritable, Viewable
         return $this->morphMany(Deletion::class, 'deletable');
     }
 
+    /**
+     * Get the references pointing from this entity to other items.
+     */
+    public function referencesFrom(): MorphMany
+    {
+        return $this->morphMany(Reference::class, 'from');
+    }
+
+    /**
+     * Get the references pointing to this entity from other items.
+     */
+    public function referencesTo(): MorphMany
+    {
+        return $this->morphMany(Reference::class, 'to');
+    }
+
     /**
      * Check if this instance or class is a certain type of entity.
      * Examples of $type are 'page', 'book', 'chapter'.
diff --git a/app/Util/CrossLinking/CrossLinkParser.php b/app/References/CrossLinkParser.php
similarity index 82%
rename from app/Util/CrossLinking/CrossLinkParser.php
rename to app/References/CrossLinkParser.php
index 774024d52..22925884a 100644
--- a/app/Util/CrossLinking/CrossLinkParser.php
+++ b/app/References/CrossLinkParser.php
@@ -1,14 +1,14 @@
 <?php
 
-namespace BookStack\Util\CrossLinking;
+namespace BookStack\References;
 
 use BookStack\Model;
-use BookStack\Util\CrossLinking\ModelResolvers\BookLinkModelResolver;
-use BookStack\Util\CrossLinking\ModelResolvers\BookshelfLinkModelResolver;
-use BookStack\Util\CrossLinking\ModelResolvers\ChapterLinkModelResolver;
-use BookStack\Util\CrossLinking\ModelResolvers\CrossLinkModelResolver;
-use BookStack\Util\CrossLinking\ModelResolvers\PageLinkModelResolver;
-use BookStack\Util\CrossLinking\ModelResolvers\PagePermalinkModelResolver;
+use BookStack\References\ModelResolvers\BookLinkModelResolver;
+use BookStack\References\ModelResolvers\BookshelfLinkModelResolver;
+use BookStack\References\ModelResolvers\ChapterLinkModelResolver;
+use BookStack\References\ModelResolvers\CrossLinkModelResolver;
+use BookStack\References\ModelResolvers\PageLinkModelResolver;
+use BookStack\References\ModelResolvers\PagePermalinkModelResolver;
 use DOMDocument;
 use DOMXPath;
 
@@ -27,7 +27,7 @@ class CrossLinkParser
     /**
      * Extract any found models within the given HTML content.
      *
-     * @returns Model[]
+     * @return Model[]
      */
     public function extractLinkedModels(string $html): array
     {
diff --git a/app/Util/CrossLinking/ModelResolvers/BookLinkModelResolver.php b/app/References/ModelResolvers/BookLinkModelResolver.php
similarity index 87%
rename from app/Util/CrossLinking/ModelResolvers/BookLinkModelResolver.php
rename to app/References/ModelResolvers/BookLinkModelResolver.php
index f2ee284cd..f33d97b84 100644
--- a/app/Util/CrossLinking/ModelResolvers/BookLinkModelResolver.php
+++ b/app/References/ModelResolvers/BookLinkModelResolver.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace BookStack\Util\CrossLinking\ModelResolvers;
+namespace BookStack\References\ModelResolvers;
 
 use BookStack\Entities\Models\Book;
 use BookStack\Model;
@@ -9,7 +9,7 @@ class BookLinkModelResolver implements CrossLinkModelResolver
 {
     public function resolve(string $link): ?Model
     {
-        $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '[#?\/$]/';
+        $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '([#?\/]|$)/';
         $matches = [];
         $match = preg_match($pattern, $link, $matches);
         if (!$match) {
diff --git a/app/Util/CrossLinking/ModelResolvers/BookshelfLinkModelResolver.php b/app/References/ModelResolvers/BookshelfLinkModelResolver.php
similarity index 88%
rename from app/Util/CrossLinking/ModelResolvers/BookshelfLinkModelResolver.php
rename to app/References/ModelResolvers/BookshelfLinkModelResolver.php
index 53cb89e3f..ca5b8ca5f 100644
--- a/app/Util/CrossLinking/ModelResolvers/BookshelfLinkModelResolver.php
+++ b/app/References/ModelResolvers/BookshelfLinkModelResolver.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace BookStack\Util\CrossLinking\ModelResolvers;
+namespace BookStack\References\ModelResolvers;
 
 use BookStack\Entities\Models\Bookshelf;
 use BookStack\Model;
@@ -9,7 +9,7 @@ class BookshelfLinkModelResolver implements CrossLinkModelResolver
 {
     public function resolve(string $link): ?Model
     {
-        $pattern = '/^' . preg_quote(url('/shelves'), '/') . '\/([\w-]+)' . '[#?\/$]/';
+        $pattern = '/^' . preg_quote(url('/shelves'), '/') . '\/([\w-]+)' . '([#?\/]|$)/';
         $matches = [];
         $match = preg_match($pattern, $link, $matches);
         if (!$match) {
diff --git a/app/Util/CrossLinking/ModelResolvers/ChapterLinkModelResolver.php b/app/References/ModelResolvers/ChapterLinkModelResolver.php
similarity index 85%
rename from app/Util/CrossLinking/ModelResolvers/ChapterLinkModelResolver.php
rename to app/References/ModelResolvers/ChapterLinkModelResolver.php
index 55afd183c..e15dba258 100644
--- a/app/Util/CrossLinking/ModelResolvers/ChapterLinkModelResolver.php
+++ b/app/References/ModelResolvers/ChapterLinkModelResolver.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace BookStack\Util\CrossLinking\ModelResolvers;
+namespace BookStack\References\ModelResolvers;
 
 use BookStack\Entities\Models\Chapter;
 use BookStack\Model;
@@ -9,7 +9,7 @@ class ChapterLinkModelResolver implements CrossLinkModelResolver
 {
     public function resolve(string $link): ?Model
     {
-        $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '\/chapter\/' . '([\w-]+)' . '[#?\/$]/';
+        $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '\/chapter\/' . '([\w-]+)' . '([#?\/]|$)/';
         $matches = [];
         $match = preg_match($pattern, $link, $matches);
         if (!$match) {
diff --git a/app/Util/CrossLinking/ModelResolvers/CrossLinkModelResolver.php b/app/References/ModelResolvers/CrossLinkModelResolver.php
similarity index 77%
rename from app/Util/CrossLinking/ModelResolvers/CrossLinkModelResolver.php
rename to app/References/ModelResolvers/CrossLinkModelResolver.php
index 073764c66..5cfd02060 100644
--- a/app/Util/CrossLinking/ModelResolvers/CrossLinkModelResolver.php
+++ b/app/References/ModelResolvers/CrossLinkModelResolver.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace BookStack\Util\CrossLinking\ModelResolvers;
+namespace BookStack\References\ModelResolvers;
 
 use BookStack\Model;
 
diff --git a/app/Util/CrossLinking/ModelResolvers/PageLinkModelResolver.php b/app/References/ModelResolvers/PageLinkModelResolver.php
similarity index 85%
rename from app/Util/CrossLinking/ModelResolvers/PageLinkModelResolver.php
rename to app/References/ModelResolvers/PageLinkModelResolver.php
index a5fea978a..f22f2734b 100644
--- a/app/Util/CrossLinking/ModelResolvers/PageLinkModelResolver.php
+++ b/app/References/ModelResolvers/PageLinkModelResolver.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace BookStack\Util\CrossLinking\ModelResolvers;
+namespace BookStack\References\ModelResolvers;
 
 use BookStack\Entities\Models\Page;
 use BookStack\Model;
@@ -9,7 +9,7 @@ class PageLinkModelResolver implements CrossLinkModelResolver
 {
     public function resolve(string $link): ?Model
     {
-        $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '\/page\/' . '([\w-]+)' . '[#?\/$]/';
+        $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '\/page\/' . '([\w-]+)' . '([#?\/]|$)/';
         $matches = [];
         $match = preg_match($pattern, $link, $matches);
         if (!$match) {
diff --git a/app/Util/CrossLinking/ModelResolvers/PagePermalinkModelResolver.php b/app/References/ModelResolvers/PagePermalinkModelResolver.php
similarity index 90%
rename from app/Util/CrossLinking/ModelResolvers/PagePermalinkModelResolver.php
rename to app/References/ModelResolvers/PagePermalinkModelResolver.php
index 9b31f5013..45396d54a 100644
--- a/app/Util/CrossLinking/ModelResolvers/PagePermalinkModelResolver.php
+++ b/app/References/ModelResolvers/PagePermalinkModelResolver.php
@@ -1,6 +1,6 @@
 <?php
 
-namespace BookStack\Util\CrossLinking\ModelResolvers;
+namespace BookStack\References\ModelResolvers;
 
 use BookStack\Entities\Models\Page;
 use BookStack\Model;
diff --git a/app/References/Reference.php b/app/References/Reference.php
new file mode 100644
index 000000000..a2a7bda10
--- /dev/null
+++ b/app/References/Reference.php
@@ -0,0 +1,26 @@
+<?php
+
+namespace BookStack\References;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\MorphTo;
+
+/**
+ * @property int $from_id
+ * @property string $from_type
+ * @property int $to_id
+ * @property string $to_type
+ */
+class Reference extends Model
+{
+    public function from(): MorphTo
+    {
+        return $this->morphTo('from');
+    }
+
+    public function to(): MorphTo
+    {
+        return $this->morphTo('to');
+    }
+}
diff --git a/app/References/ReferenceService.php b/app/References/ReferenceService.php
new file mode 100644
index 000000000..7a1cf2fed
--- /dev/null
+++ b/app/References/ReferenceService.php
@@ -0,0 +1,71 @@
+<?php
+
+namespace BookStack\References;
+
+use BookStack\Entities\Models\Page;
+use Illuminate\Database\Eloquent\Collection;
+
+class ReferenceService
+{
+
+    /**
+     * Update the outgoing references for the given page.
+     */
+    public function updateForPage(Page $page): void
+    {
+        $this->updateForPages([$page]);
+    }
+
+    /**
+     * Update the outgoing references for all pages in the system.
+     */
+    public function updateForAllPages(): void
+    {
+        Reference::query()
+            ->where('from_type', '=', (new Page())->getMorphClass())
+            ->truncate();
+
+        Page::query()->select(['id', 'html'])->chunk(100, function(Collection $pages) {
+            $this->updateForPages($pages->all());
+        });
+    }
+
+    /**
+     * Update the outgoing references for the pages in the given array.
+     *
+     * @param Page[] $pages
+     */
+    protected function updateForPages(array $pages): void
+    {
+        if (count($pages) === 0) {
+            return;
+        }
+
+        $parser = CrossLinkParser::createWithEntityResolvers();
+        $references = [];
+
+        $pageIds = array_map(fn(Page $page) => $page->id, $pages);
+        Reference::query()
+            ->where('from_type', '=', $pages[0]->getMorphClass())
+            ->whereIn('from_id', $pageIds)
+            ->delete();
+
+        foreach ($pages as $page) {
+            $models = $parser->extractLinkedModels($page->html);
+
+            foreach ($models as $model) {
+                $references[] = [
+                    'from_id' => $page->id,
+                    'from_type' => $page->getMorphClass(),
+                    'to_id' => $model->id,
+                    'to_type' => $model->getMorphClass(),
+                ];
+            }
+        }
+
+        foreach (array_chunk($references, 1000) as $referenceDataChunk) {
+            Reference::query()->insert($referenceDataChunk);
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/database/migrations/2022_08_17_092941_create_references_table.php b/database/migrations/2022_08_17_092941_create_references_table.php
new file mode 100644
index 000000000..443bce551
--- /dev/null
+++ b/database/migrations/2022_08_17_092941_create_references_table.php
@@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateReferencesTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('references', function (Blueprint $table) {
+            $table->id();
+            $table->unsignedInteger('from_id')->index();
+            $table->string('from_type', 25)->index();
+            $table->unsignedInteger('to_id')->index();
+            $table->string('to_type', 25)->index();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('references');
+    }
+}
diff --git a/tests/Util/CrossLinkParserTest.php b/tests/References/CrossLinkParserTest.php
similarity index 67%
rename from tests/Util/CrossLinkParserTest.php
rename to tests/References/CrossLinkParserTest.php
index f8ad59db2..42d78cb0a 100644
--- a/tests/Util/CrossLinkParserTest.php
+++ b/tests/References/CrossLinkParserTest.php
@@ -1,9 +1,10 @@
 <?php
 
-namespace Tests\Util;
+namespace Tests\References;
 
+use BookStack\Entities\Models\Book;
 use BookStack\Entities\Models\Page;
-use BookStack\Util\CrossLinking\CrossLinkParser;
+use BookStack\References\CrossLinkParser;
 use Tests\TestCase;
 
 class CrossLinkParserTest extends TestCase
@@ -38,4 +39,24 @@ class CrossLinkParserTest extends TestCase
         $this->assertEquals(get_class($entities['bookshelf']), get_class($results[4]));
         $this->assertEquals($entities['bookshelf']->id, $results[4]->id);
     }
+
+    public function test_similar_page_and_book_reference_links_dont_conflict()
+    {
+        $page = Page::query()->first();
+        $book = $page->book;
+
+        $html = '
+<a href="' . $page->getUrl() . '">Page Link</a>
+<a href="' . $book->getUrl() . '">Book Link</a>
+        ';
+
+        $parser = CrossLinkParser::createWithEntityResolvers();
+        $results = $parser->extractLinkedModels($html);
+
+        $this->assertCount(2, $results);
+        $this->assertEquals(get_class($page), get_class($results[0]));
+        $this->assertEquals($page->id, $results[0]->id);
+        $this->assertEquals(get_class($book), get_class($results[1]));
+        $this->assertEquals($book->id, $results[1]->id);
+    }
 }