Database collations (#489)

* SyncDatabases

* Collation on Create inc WordPress

* Refactored Enum

* Resolve sync issue

* Fix for PostgreSQL

* pint

* reversed enum

* style adjustments

* add unit tests

* style

* fix tests

* more tests

---------

Co-authored-by: Saeed Vaziry <61919774+saeedvaziry@users.noreply.github.com>
Co-authored-by: Saeed Vaziry <mr.saeedvaziry@gmail.com>

app/Actions/Database/CreateDatabase.php

@@ -14,11 +14,13 @@ public function create(Server $server, array $input): Database
     {
         $database = new Database([
             'server_id' => $server->id,
+            'charset' => $input['charset'],
+            'collation' => $input['collation'],
             'name' => $input['name'],
         ]);
         /** @var \App\SSH\Services\Database\Database $databaseHandler */
         $databaseHandler = $server->database()->handler();
-        $databaseHandler->create($database->name);
+        $databaseHandler->create($database->name, $database->charset, $database->collation);
         $database->status = DatabaseStatus::READY;
         $database->save();
 
@@ -42,6 +44,14 @@ public static function rules(Server $server, array $input): array
                 'alpha_dash',
                 Rule::unique('databases', 'name')->where('server_id', $server->id),
             ],
+            'charset' => [
+                'required',
+                'string',
+            ],
+            'collation' => [
+                'required',
+                'string',
+            ],
         ];
         if (isset($input['user']) && $input['user']) {
             $rules['username'] = [

app/Http/Controllers/API/DatabaseController.php

@@ -41,6 +41,8 @@ public function index(Project $project, Server $server): ResourceCollection
     #[Post('/', name: 'api.projects.servers.databases.create', middleware: 'ability:write')]
     #[Endpoint(title: 'create', description: 'Create a new database.')]
     #[BodyParam(name: 'name', required: true)]
+    #[BodyParam(name: 'charset', required: true)]
+    #[BodyParam(name: 'collation', required: true)]
     #[ResponseFromApiResource(DatabaseResource::class, Database::class)]
     public function create(Request $request, Project $project, Server $server): DatabaseResource
     {

app/Models/Database.php

@@ -12,6 +12,8 @@
 /**
  * @property int $server_id
  * @property string $name
+ * @property string $collation
+ * @property string $charset
  * @property string $status
  * @property Server $server
  * @property Backup[] $backups
@@ -25,6 +27,8 @@ class Database extends AbstractModel
     protected $fillable = [
         'server_id',
         'name',
+        'collation',
+        'charset',
         'status',
     ];
 

app/SSH/Services/Database/AbstractDatabase.php

@@ -11,6 +11,16 @@
 
 abstract class AbstractDatabase extends AbstractService implements Database
 {
+    protected array $systemDbs = [];
+
+    protected string $defaultCharset;
+
+    protected string $separator = "\t";
+
+    protected int $headerLines = 1;
+
+    protected bool $removeLastRow = false;
+
     protected function getScriptView(string $script): string
     {
         return 'ssh.services.database.'.$this->service->name.'.'.$script;
@@ -43,6 +53,9 @@ public function install(): void
         $status = $this->service->server->systemd()->status($this->service->unit);
         $this->service->validateInstall($status);
         $this->service->server->os()->cleanup();
+
+        $this->updateCharsets();
+        $this->syncDatabases();
     }
 
     public function deletionRules(): array
@@ -83,11 +96,13 @@ public function uninstall(): void
     /**
      * @throws SSHError
      */
-    public function create(string $name): void
+    public function create(string $name, string $charset, string $collation): void
     {
         $this->service->server->ssh()->exec(
             view($this->getScriptView('create'), [
                 'name' => $name,
+                'charset' => $charset,
+                'collation' => $collation,
             ]),
             'create-database'
         );
@@ -219,4 +234,124 @@ public function restoreBackup(BackupFile $backupFile, string $database): void
             'restore-database'
         );
     }
+
+    /**
+     * @throws SSHError
+     */
+    public function updateCharsets(): void
+    {
+        $data = $this->service->server->ssh()->exec(
+            view($this->getScriptView('get-charsets')),
+            'get-database-charsets'
+        );
+
+        $charsets = $this->tableToArray($data);
+
+        $results = [];
+        $charsetCollations = [];
+
+        foreach ($charsets as $key => $charset) {
+            if (empty($charsetCollations[$charset[1]])) {
+                $charsetCollations[$charset[1]] = [];
+            }
+
+            $charsetCollations[$charset[1]][] = $charset[0];
+
+            if ($charset[3] === 'Yes') {
+                $results[$charset[1]] = [
+                    'default' => $charset[0],
+                    'list' => [],
+                ];
+
+                continue;
+            }
+
+            if ($key == count($charsets) - 1) {
+                $results[$charset[1]] = [
+                    'default' => null,
+                    'list' => [],
+                ];
+            }
+        }
+
+        foreach ($results as $charset => $data) {
+            $results[$charset]['list'] = $charsetCollations[$charset];
+        }
+
+        ksort($results);
+
+        $data = array_merge(
+            $this->service->type_data ?? [],
+            ['charsets' => $results, 'defaultCharset' => $this->defaultCharset]
+        );
+
+        $this->service->update(['type_data' => $data]);
+
+    }
+
+    /**
+     * @throws SSHError
+     */
+    public function syncDatabases(bool $createNew = true): void
+    {
+        $data = $this->service->server->ssh()->exec(
+            view($this->getScriptView('get-db-list')),
+            'get-db-list'
+        );
+
+        $databases = $this->tableToArray($data);
+
+        foreach ($databases as $database) {
+            if (in_array($database[0], $this->systemDbs)) {
+                continue;
+            }
+
+            $db = $this->service->server->databases()
+                ->where('name', $database[0])
+                ->first();
+
+            if ($db === null) {
+                if ($createNew) {
+                    $this->service->server->databases()->create([
+                        'name' => $database[0],
+                        'collation' => $database[2],
+                        'charset' => $database[1],
+                    ]);
+                }
+
+                continue;
+            }
+
+            if ($db->collation !== $database[2] || $db->charset !== $database[1]) {
+                $db->update([
+                    'collation' => $database[2],
+                    'charset' => $database[1],
+                ]);
+            }
+        }
+    }
+
+    protected function tableToArray(string $data, bool $keepHeader = false): array
+    {
+        $lines = explode("\n", trim($data));
+
+        if (! $keepHeader) {
+            for ($i = 0; $i < $this->headerLines; $i++) {
+                array_shift($lines);
+            }
+        }
+
+        if ($this->removeLastRow) {
+            array_pop($lines);
+        }
+
+        $rows = [];
+        foreach ($lines as $line) {
+            $row = explode($this->separator, $line);
+            $row = array_map('trim', $row);
+            $rows[] = $row;
+        }
+
+        return $rows;
+    }
 }

app/SSH/Services/Database/Database.php

@@ -6,7 +6,7 @@
 
 interface Database
 {
-    public function create(string $name): void;
+    public function create(string $name, string $charset, string $collation): void;
 
     public function delete(string $name): void;
 
@@ -21,4 +21,8 @@ public function unlink(string $username, string $host): void;
     public function runBackup(BackupFile $backupFile): void;
 
     public function restoreBackup(BackupFile $backupFile, string $database): void;
+
+    public function updateCharsets(): void;
+
+    public function syncDatabases(bool $createNew = true): void;
 }

app/SSH/Services/Database/Mariadb.php

@@ -4,5 +4,7 @@
 
 class Mariadb extends AbstractDatabase
 {
-    //
+    protected array $systemDbs = ['information_schema', 'performance_schema', 'mysql', 'sys'];
+
+    protected string $defaultCharset = 'utf8mb3';
 }

app/SSH/Services/Database/Mysql.php

@@ -4,5 +4,7 @@
 
 class Mysql extends AbstractDatabase
 {
-    //
+    protected array $systemDbs = ['information_schema', 'performance_schema', 'mysql', 'sys'];
+
+    protected string $defaultCharset = 'utf8mb3';
 }

app/SSH/Services/Database/Postgresql.php

@@ -4,5 +4,13 @@
 
 class Postgresql extends AbstractDatabase
 {
-    //
+    protected array $systemDbs = ['template0', 'template1', 'postgres'];
+
+    protected string $defaultCharset = 'UTF8';
+
+    protected int $headerLines = 2;
+
+    protected string $separator = '|';
+
+    protected bool $removeLastRow = true;
 }

app/SiteTypes/Wordpress.php

@@ -80,6 +80,8 @@ public function data(array $input): array
             'email' => $input['email'],
             'password' => $input['password'],
             'database' => $input['database'],
+            'database_charset' => $input['charset'],
+            'database_collation' => $input['collation'],
             'database_user' => $input['database_user'],
             'database_password' => $input['database_password'],
         ];
@@ -96,20 +98,28 @@ public function install(): void
         $webserver = $this->site->server->webserver()->handler();
         $webserver->createVHost($this->site);
         $this->progress(30);
+
         /** @var Database $database */
         $database = app(CreateDatabase::class)->create($this->site->server, [
             'name' => $this->site->type_data['database'],
+            'charset' => $this->site->type_data['database_charset'],
+            'collation' => $this->site->type_data['database_collation'],
         ]);
+
         /** @var DatabaseUser $databaseUser */
         $databaseUser = app(CreateDatabaseUser::class)->create($this->site->server, [
             'username' => $this->site->type_data['database_user'],
             'password' => $this->site->type_data['database_password'],
+            'collation' => $this->site->type_data['database_collation'],
+            'charset' => $this->site->type_data['database_charset'],
             'remote' => false,
             'host' => 'localhost',
         ], [$database->name]);
+
         app(LinkUser::class)->link($databaseUser, [
             'databases' => [$database->name],
         ]);
+
         $this->site->php()?->restart();
         $this->progress(60);
         app(\App\SSH\Wordpress\Wordpress::class)->install($this->site);

app/Web/Pages/Servers/Databases/Index.php

@@ -4,11 +4,15 @@
 
 use App\Actions\Database\CreateDatabase;
 use App\Models\Database;
+use App\Models\Server;
 use App\Web\Contracts\HasSecondSubNav;
 use App\Web\Pages\Servers\Page;
 use Filament\Actions\Action;
 use Filament\Forms\Components\Checkbox;
+use Filament\Forms\Components\Select;
 use Filament\Forms\Components\TextInput;
+use Filament\Forms\Get;
+use Filament\Forms\Set;
 use Filament\Notifications\Notification;
 use Filament\Support\Enums\MaxWidth;
 
@@ -25,6 +29,59 @@ public function mount(): void
         $this->authorize('viewAny', [Database::class, $this->server]);
     }
 
+    public static function getCharsetInput(Server $server): Select
+    {
+        return Select::make('charset')
+            ->label('Charset / Encoding')
+            ->native(false)
+            ->live()
+            ->default(function () use ($server) {
+                $service = $server->defaultService('database');
+
+                return $service->type_data['defaultCharset'] ?? null;
+            })
+            ->options(function () use ($server) {
+                $service = $server->defaultService('database');
+                $charsets = $service->type_data['charsets'] ?? [];
+
+                return array_combine(
+                    array_keys($charsets),
+                    array_keys($charsets)
+                );
+            })
+            ->afterStateUpdated(function (Get $get, Set $set, $state) use ($server) {
+                $service = $server->defaultService('database');
+                $charsets = $service->type_data['charsets'] ?? [];
+                $set('collation', $charsets[$state]['default'] ?? null);
+            })
+            ->rules(fn (callable $get) => CreateDatabase::rules($server, $get())['charset']);
+    }
+
+    public static function getCollationInput(Server $server): Select
+    {
+        return Select::make('collation')
+            ->label('Collation')
+            ->native(false)
+            ->live()
+            ->default(function (Get $get) use ($server) {
+                $service = $server->defaultService('database');
+                $charsets = $service->type_data['charsets'] ?? [];
+                $charset = $get('charset') ?? $service->type_data['default'] ?? 'utf8mb4';
+
+                return $charsets[$charset]['default'] ?? null;
+            })
+            ->options(function (Get $get) use ($server) {
+                $service = $server->defaultService('database');
+                $collations = $service->type_data['charsets'][$get('charset')]['list'] ?? [];
+
+                return array_combine(
+                    array_values($collations),
+                    array_values($collations)
+                );
+            })
+            ->rules(fn (callable $get) => CreateDatabase::rules($server, $get())['collation']);
+    }
+
     protected function getHeaderActions(): array
     {
         return [
@@ -36,6 +93,8 @@ protected function getHeaderActions(): array
                 ->form([
                     TextInput::make('name')
                         ->rules(fn (callable $get) => CreateDatabase::rules($this->server, $get())['name']),
+                    self::getCharsetInput($this->server),
+                    self::getCollationInput($this->server),
                     Checkbox::make('user')
                         ->label('Create User')
                         ->default(false)

app/Web/Pages/Servers/Databases/Widgets/DatabasesList.php

@@ -27,6 +27,12 @@ protected function getTableColumns(): array
         return [
             TextColumn::make('name')
                 ->searchable(),
+            TextColumn::make('charset')
+                ->label('Charset / Encoding')
+                ->sortable(),
+            TextColumn::make('collation')
+                ->label('Collation')
+                ->sortable(),
             TextColumn::make('status')
                 ->label('Status')
                 ->badge()

app/Web/Pages/Servers/Sites/Index.php

@@ -199,6 +199,8 @@ private function wordpressFields(): Component
                 TextInput::make('database')
                     ->helperText('It will create a database with this name')
                     ->rules(fn (Get $get) => CreateSite::rules($this->server, $get())['database']),
+                \App\Web\Pages\Servers\Databases\Index::getCharsetInput($this->server),
+                \App\Web\Pages\Servers\Databases\Index::getCollationInput($this->server),
                 TextInput::make('database_user')
                     ->helperText('It will create a db user with this username')
                     ->rules(fn (Get $get) => CreateSite::rules($this->server, $get())['database']),