From ea3f011f34767888b090ac527315b01c68f42b0c Mon Sep 17 00:00:00 2001 From: Saeed Vaziry Date: Fri, 25 Aug 2023 21:05:18 +0200 Subject: [PATCH] database backups --- .env.prod | 2 +- .env.testing | 6 +- .../{Backup => Database}/CreateBackup.php | 38 +++---- .../StorageProvider/AddStorageProvider.php | 30 ----- .../StorageProvider/CreateStorageProvider.php | 52 +++++++++ .../HandleProviderCallback.php | 34 ------ .../StorageProvider/ValidateProvider.php | 28 ----- app/Console/Commands/RunBackup.php | 49 ++++++++ app/Console/Kernel.php | 5 +- app/Contracts/StorageProvider.php | 2 +- app/Enums/BackupFileStatus.php | 22 ++++ app/Enums/BackupStatus.php | 16 +++ app/Exceptions/BackupFileException.php | 9 ++ app/Http/Controllers/DatabaseController.php | 12 +- .../Databases/DatabaseBackupFiles.php | 67 +++++++++++ .../Livewire/Databases/DatabaseBackups.php | 81 ++++++++++++++ .../StorageProviders/ConnectProvider.php | 30 +++++ .../StorageProviders/ProvidersList.php | 37 ++++++ app/Jobs/Backup/RestoreDatabase.php | 5 +- app/Jobs/Backup/RunBackup.php | 5 +- app/Models/Backup.php | 7 +- app/Models/BackupFile.php | 5 +- app/Models/StorageProvider.php | 18 +-- app/ServiceHandlers/Database/Mysql.php | 2 +- app/StorageProviders/Dropbox.php | 32 ++++-- database/factories/BackupFactory.php | 6 +- database/factories/StorageProviderFactory.php | 6 +- ..._095440_update_storage_providers_table.php | 34 ++++++ ...2023_08_17_231824_update_backups_table.php | 21 ++++ ...08_25_183201_update_backup_files_table.php | 21 ++++ install/install.sh | 3 + package.json | 3 +- .../commands/storage/upload-to-dropbox.sh | 2 +- resources/views/databases/backups.blade.php | 7 ++ resources/views/databases/index.blade.php | 2 + resources/views/layouts/app.blade.php | 2 +- resources/views/layouts/profile.blade.php | 6 + resources/views/livewire/broadcast.blade.php | 2 +- .../databases/database-backup-files.blade.php | 74 ++++++++++++ .../databases/database-backups.blade.php | 49 ++++++++ .../databases/database-list.blade.php | 8 +- .../databases/database-user-list.blade.php | 10 +- .../partials/backup-file-status.blade.php | 21 ++++ .../partials/backup-status.blade.php | 9 ++ .../partials/create-backup-modal.blade.php | 78 +++++++++++++ .../delete-backup-file-modal.blade.php | 6 + .../partials/delete-backup-modal.blade.php | 6 + .../partials/restore-backup-modal.blade.php | 30 +++++ .../connect-provider.blade.php | 54 +++++++++ .../providers-list.blade.php | 45 ++++++++ .../views/server-providers/index.blade.php | 2 +- .../views/storage-providers/index.blade.php | 5 + routes/web.php | 4 + tests/Feature/Http/DatabaseBackupTest.php | 105 ++++++++++++++++++ tests/Feature/Http/StorageProvidersTest.php | 68 ++++++++++++ 55 files changed, 1111 insertions(+), 172 deletions(-) rename app/Actions/{Backup => Database}/CreateBackup.php (61%) delete mode 100644 app/Actions/StorageProvider/AddStorageProvider.php create mode 100644 app/Actions/StorageProvider/CreateStorageProvider.php delete mode 100755 app/Actions/StorageProvider/HandleProviderCallback.php delete mode 100755 app/Actions/StorageProvider/ValidateProvider.php create mode 100644 app/Console/Commands/RunBackup.php create mode 100644 app/Enums/BackupFileStatus.php create mode 100644 app/Enums/BackupStatus.php create mode 100644 app/Exceptions/BackupFileException.php create mode 100644 app/Http/Livewire/Databases/DatabaseBackupFiles.php create mode 100644 app/Http/Livewire/Databases/DatabaseBackups.php create mode 100644 app/Http/Livewire/StorageProviders/ConnectProvider.php create mode 100644 app/Http/Livewire/StorageProviders/ProvidersList.php create mode 100644 database/migrations/2023_08_13_095440_update_storage_providers_table.php create mode 100644 database/migrations/2023_08_17_231824_update_backups_table.php create mode 100644 database/migrations/2023_08_25_183201_update_backup_files_table.php create mode 100644 resources/views/databases/backups.blade.php create mode 100644 resources/views/livewire/databases/database-backup-files.blade.php create mode 100644 resources/views/livewire/databases/database-backups.blade.php create mode 100644 resources/views/livewire/databases/partials/backup-file-status.blade.php create mode 100644 resources/views/livewire/databases/partials/backup-status.blade.php create mode 100644 resources/views/livewire/databases/partials/create-backup-modal.blade.php create mode 100644 resources/views/livewire/databases/partials/delete-backup-file-modal.blade.php create mode 100644 resources/views/livewire/databases/partials/delete-backup-modal.blade.php create mode 100644 resources/views/livewire/databases/partials/restore-backup-modal.blade.php create mode 100644 resources/views/livewire/storage-providers/connect-provider.blade.php create mode 100644 resources/views/livewire/storage-providers/providers-list.blade.php create mode 100644 resources/views/storage-providers/index.blade.php create mode 100644 tests/Feature/Http/DatabaseBackupTest.php create mode 100644 tests/Feature/Http/StorageProvidersTest.php diff --git a/.env.prod b/.env.prod index ea8d52b..00ed9e8 100755 --- a/.env.prod +++ b/.env.prod @@ -15,7 +15,7 @@ DB_USERNAME= DB_PASSWORD= BROADCAST_DRIVER=pusher -CACHE_DRIVER=redis +CACHE_DRIVER=file FILESYSTEM_DRIVER=local QUEUE_CONNECTION=default SESSION_DRIVER=database diff --git a/.env.testing b/.env.testing index a919ab1..7911a44 100755 --- a/.env.testing +++ b/.env.testing @@ -8,11 +8,11 @@ LOG_CHANNEL=stack LOG_LEVEL=debug DB_CONNECTION=mysql -DB_HOST=mysql +DB_HOST=127.0.0.1 DB_PORT=3306 -DB_DATABASE=testing +DB_DATABASE=vito_test DB_USERNAME=root -DB_PASSWORD=password +DB_PASSWORD= BROADCAST_DRIVER=pusher CACHE_DRIVER=array diff --git a/app/Actions/Backup/CreateBackup.php b/app/Actions/Database/CreateBackup.php similarity index 61% rename from app/Actions/Backup/CreateBackup.php rename to app/Actions/Database/CreateBackup.php index e2ee9fa..b8d6edc 100644 --- a/app/Actions/Backup/CreateBackup.php +++ b/app/Actions/Database/CreateBackup.php @@ -1,14 +1,12 @@ validate($type, $server, $user, $input); - - if ($type == 'database') { - Gate::forUser($user)->authorize('viewAny', [Database::class, $server]); - } + $this->validate($type, $server, $input); $backup = new Backup([ - 'name' => $input['name'], 'type' => $type, 'server_id' => $server->id, 'database_id' => $input['database'] ?? null, 'storage_id' => $input['storage'], - 'interval' => $input['interval'], - 'keep_backups' => $input['keep_backups'], - 'status' => 'running', + 'interval' => $input['interval'] == 'custom' ? $input['custom'] : $input['interval'], + 'keep_backups' => $input['keep'], + 'status' => BackupStatus::RUNNING, ]); $backup->save(); @@ -47,17 +40,14 @@ public function create($type, Server $server, User $user, array $input): Backup /** * @throws ValidationException */ - private function validate($type, Server $server, User $user, array $input): void + private function validate($type, Server $server, array $input): void { $rules = [ - 'name' => 'required', 'storage' => [ 'required', - Rule::exists('storage_providers', 'id') - ->where('user_id', $user->id) - ->where('connected', 1), + Rule::exists('storage_providers', 'id'), ], - 'keep_backups' => [ + 'keep' => [ 'required', 'numeric', 'min:1', @@ -69,9 +59,15 @@ private function validate($type, Server $server, User $user, array $input): void '0 0 * * *', '0 0 * * 0', '0 0 1 * *', + 'custom' ]), ], ]; + if ($input['interval'] == 'custom') { + $rules['custom'] = [ + 'required', + ]; + } if ($type === 'database') { $rules['database'] = [ 'required', @@ -80,6 +76,6 @@ private function validate($type, Server $server, User $user, array $input): void ->where('status', DatabaseStatus::READY), ]; } - Validator::make($input, $rules)->validateWithBag('createBackup'); + Validator::make($input, $rules)->validate(); } } diff --git a/app/Actions/StorageProvider/AddStorageProvider.php b/app/Actions/StorageProvider/AddStorageProvider.php deleted file mode 100644 index b2a02b9..0000000 --- a/app/Actions/StorageProvider/AddStorageProvider.php +++ /dev/null @@ -1,30 +0,0 @@ -validate($user, $input); - - $storageProvider = new StorageProvider([ - 'user_id' => $user->id, - 'provider' => $input['provider'], - 'label' => $input['label'], - 'connected' => false, - ]); - $storageProvider->save(); - - return $storageProvider->provider()->connect(); - } -} diff --git a/app/Actions/StorageProvider/CreateStorageProvider.php b/app/Actions/StorageProvider/CreateStorageProvider.php new file mode 100644 index 0000000..f70a767 --- /dev/null +++ b/app/Actions/StorageProvider/CreateStorageProvider.php @@ -0,0 +1,52 @@ +validate($user, $input); + + $storageProvider = new StorageProvider([ + 'user_id' => $user->id, + 'provider' => $input['provider'], + 'profile' => $input['name'], + 'credentials' => [ + 'token' => $input['token'] + ] + ]); + if (! $storageProvider->provider()->connect()) { + throw ValidationException::withMessages([ + 'token' => __("Couldn't connect to the provider") + ]); + } + $storageProvider->save();; + } + + private function validate(User $user, array $input): void + { + Validator::make($input, [ + 'provider' => [ + 'required', + Rule::in(config('core.storage_providers')), + ], + 'name' => [ + 'required', + Rule::unique('storage_providers', 'profile')->where('user_id', $user->id), + ], + 'token' => [ + 'required' + ] + ])->validate(); + } +} diff --git a/app/Actions/StorageProvider/HandleProviderCallback.php b/app/Actions/StorageProvider/HandleProviderCallback.php deleted file mode 100755 index 5242e2c..0000000 --- a/app/Actions/StorageProvider/HandleProviderCallback.php +++ /dev/null @@ -1,34 +0,0 @@ -session()->get('storage_provider_id'); - /** @var StorageProvider $storageProvider */ - $storageProvider = StorageProvider::query()->findOrFail($providerId); - /** @var User $oauthUser */ - $oauthUser = Socialite::driver($provider)->user(); - $storageProvider->token = $oauthUser->token; - $storageProvider->refresh_token = $oauthUser->refreshToken; - $storageProvider->token_expires_at = now()->addSeconds($oauthUser->expiresIn); - $storageProvider->connected = true; - $storageProvider->save(); - /** @TODO toast success message */ - } catch (Throwable) { - /** @TODO toast failed message */ - } - - return redirect()->route('storage-providers'); - } -} diff --git a/app/Actions/StorageProvider/ValidateProvider.php b/app/Actions/StorageProvider/ValidateProvider.php deleted file mode 100755 index 6b0ca05..0000000 --- a/app/Actions/StorageProvider/ValidateProvider.php +++ /dev/null @@ -1,28 +0,0 @@ - [ - 'required', - Rule::unique('storage_providers', 'label')->where('user_id', $user->id), - ], - 'provider' => [ - 'required', - Rule::in(config('core.storage_providers')), - ], - ])->validateWithBag('addStorageProvider'); - } -} diff --git a/app/Console/Commands/RunBackup.php b/app/Console/Commands/RunBackup.php new file mode 100644 index 0000000..5d8df57 --- /dev/null +++ b/app/Console/Commands/RunBackup.php @@ -0,0 +1,49 @@ +where('interval', $this->argument('interval')) + ->where('status', 'running') + ->chunk(100, function ($backups) { + /** @var Backup $backup */ + foreach ($backups as $backup) { + $backup->run(); + } + }); + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index e6b9960..59083da 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -12,7 +12,10 @@ class Kernel extends ConsoleKernel */ protected function schedule(Schedule $schedule): void { - // $schedule->command('inspire')->hourly(); + $schedule->command('backups:run "0 * * * *"')->hourly(); + $schedule->command('backups:run "0 0 * * *"')->daily(); + $schedule->command('backups:run "0 0 * * 0"')->weekly(); + $schedule->command('backups:run "0 0 1 * *"')->monthly(); } /** diff --git a/app/Contracts/StorageProvider.php b/app/Contracts/StorageProvider.php index 0fe66f9..fef2d84 100644 --- a/app/Contracts/StorageProvider.php +++ b/app/Contracts/StorageProvider.php @@ -7,7 +7,7 @@ interface StorageProvider { - public function connect(): RedirectResponse; + public function connect(): bool; public function upload(Server $server, string $src, string $dest): array; diff --git a/app/Enums/BackupFileStatus.php b/app/Enums/BackupFileStatus.php new file mode 100644 index 0000000..9b2e59c --- /dev/null +++ b/app/Enums/BackupFileStatus.php @@ -0,0 +1,22 @@ + $server, ]); } + + public function backups(Server $server, Backup $backup): View + { + return view('databases.backups', [ + 'server' => $server, + 'backup' => $backup, + ]); + } } diff --git a/app/Http/Livewire/Databases/DatabaseBackupFiles.php b/app/Http/Livewire/Databases/DatabaseBackupFiles.php new file mode 100644 index 0000000..1c0660a --- /dev/null +++ b/app/Http/Livewire/Databases/DatabaseBackupFiles.php @@ -0,0 +1,67 @@ +backup->run(); + + $this->refreshComponent([]); + } + + public function restore(): void + { + /** @var BackupFile $file */ + $file = BackupFile::query()->findOrFail($this->restoreId); + + /** @var Database $database */ + $database = Database::query()->findOrFail($this->restoreDatabaseId); + + $file->restore($database); + + $this->refreshComponent([]); + + $this->dispatchBrowserEvent('restored', true); + } + + public function delete(): void + { + /** @var BackupFile $file */ + $file = BackupFile::query()->findOrFail($this->deleteId); + + $file->delete(); + + $this->dispatchBrowserEvent('confirmed', true); + } + + public function render(): View + { + return view('livewire.databases.database-backup-files', [ + 'files' => $this->backup->files()->orderByDesc('id')->simplePaginate(10) + ]); + } +} diff --git a/app/Http/Livewire/Databases/DatabaseBackups.php b/app/Http/Livewire/Databases/DatabaseBackups.php new file mode 100644 index 0000000..d3f85c8 --- /dev/null +++ b/app/Http/Livewire/Databases/DatabaseBackups.php @@ -0,0 +1,81 @@ +create('database', $this->server, $this->all()); + + $this->refreshComponent([]); + + $this->dispatchBrowserEvent('backup-created', true); + } + + public function files(int $id): void + { + $backup = Backup::query()->findOrFail($id); + $this->backup = $backup; + $this->files = $backup->files()->orderByDesc('id')->simplePaginate(1); + $this->dispatchBrowserEvent('show-files', true); + } + + public function backup(): void + { + $this->backup?->run(); + + $this->files = $this->backup?->files()->orderByDesc('id')->simplePaginate(); + + $this->dispatchBrowserEvent('backup-running', true); + } + + public function delete(): void + { + /** @var Backup $backup */ + $backup = Backup::query()->findOrFail($this->deleteId); + + $backup->delete(); + + $this->dispatchBrowserEvent('confirmed', true); + } + + public function render(): View + { + return view('livewire.databases.database-backups', [ + 'backups' => $this->server->backups, + 'databases' => $this->server->databases, + 'files' => $this->files + ]); + } +} diff --git a/app/Http/Livewire/StorageProviders/ConnectProvider.php b/app/Http/Livewire/StorageProviders/ConnectProvider.php new file mode 100644 index 0000000..d9bbb5d --- /dev/null +++ b/app/Http/Livewire/StorageProviders/ConnectProvider.php @@ -0,0 +1,30 @@ +create(auth()->user(), $this->all()); + + $this->emitTo(ProvidersList::class, '$refresh'); + + $this->dispatchBrowserEvent('connected', true); + } + + public function render(): View + { + return view('livewire.storage-providers.connect-provider'); + } +} diff --git a/app/Http/Livewire/StorageProviders/ProvidersList.php b/app/Http/Livewire/StorageProviders/ProvidersList.php new file mode 100644 index 0000000..2370b3e --- /dev/null +++ b/app/Http/Livewire/StorageProviders/ProvidersList.php @@ -0,0 +1,37 @@ +findOrFail($this->deleteId); + + $provider->delete(); + + $this->refreshComponent([]); + + $this->dispatchBrowserEvent('confirmed', true); + } + + public function render(): View + { + return view('livewire.storage-providers.providers-list', [ + 'providers' => StorageProvider::query()->latest()->get(), + ]); + } +} diff --git a/app/Jobs/Backup/RestoreDatabase.php b/app/Jobs/Backup/RestoreDatabase.php index bc3144e..081e16a 100644 --- a/app/Jobs/Backup/RestoreDatabase.php +++ b/app/Jobs/Backup/RestoreDatabase.php @@ -2,6 +2,7 @@ namespace App\Jobs\Backup; +use App\Enums\BackupFileStatus; use App\Events\Broadcast; use App\Jobs\Job; use App\Models\BackupFile; @@ -23,7 +24,7 @@ public function handle(): void { $this->database->server->database()->handler()->restoreBackup($this->backupFile, $this->database->name); - $this->backupFile->status = 'restored'; + $this->backupFile->status = BackupFileStatus::RESTORED; $this->backupFile->restored_at = now(); $this->backupFile->save(); @@ -36,7 +37,7 @@ public function handle(): void public function failed(): void { - $this->backupFile->status = 'restore_failed'; + $this->backupFile->status = BackupFileStatus::RESTORE_FAILED; $this->backupFile->save(); event( new Broadcast('backup-restore-failed', [ diff --git a/app/Jobs/Backup/RunBackup.php b/app/Jobs/Backup/RunBackup.php index f45bb93..d08b755 100644 --- a/app/Jobs/Backup/RunBackup.php +++ b/app/Jobs/Backup/RunBackup.php @@ -2,6 +2,7 @@ namespace App\Jobs\Backup; +use App\Enums\BackupFileStatus; use App\Events\Broadcast; use App\Jobs\Job; use App\Models\BackupFile; @@ -21,7 +22,7 @@ public function handle(): void $this->backupFile->backup->server->database()->handler()->runBackup($this->backupFile); } - $this->backupFile->status = 'finished'; + $this->backupFile->status = BackupFileStatus::CREATED; $this->backupFile->save(); event( @@ -33,7 +34,7 @@ public function handle(): void public function failed(): void { - $this->backupFile->status = 'failed'; + $this->backupFile->status = BackupFileStatus::FAILED; $this->backupFile->save(); event( new Broadcast('run-backup-failed', [ diff --git a/app/Models/Backup.php b/app/Models/Backup.php index 422e251..2a8c382 100644 --- a/app/Models/Backup.php +++ b/app/Models/Backup.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Enums\BackupFileStatus; use App\Jobs\Backup\RunBackup; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\BelongsTo; @@ -10,7 +11,6 @@ /** * @property string $type - * @property string $name * @property int $server_id * @property int $storage_id * @property int $database_id @@ -28,7 +28,6 @@ class Backup extends AbstractModel protected $fillable = [ 'type', - 'name', 'server_id', 'storage_id', 'database_id', @@ -77,8 +76,8 @@ public function run(): void { $file = new BackupFile([ 'backup_id' => $this->id, - 'name' => Str::of($this->name)->slug().'-'.now()->format('YmdHis'), - 'status' => 'creating', + 'name' => Str::of($this->database->name)->slug().'-'.now()->format('YmdHis'), + 'status' => BackupFileStatus::CREATING, ]); $file->save(); diff --git a/app/Models/BackupFile.php b/app/Models/BackupFile.php index c62a30a..979f325 100644 --- a/app/Models/BackupFile.php +++ b/app/Models/BackupFile.php @@ -2,6 +2,7 @@ namespace App\Models; +use App\Enums\BackupFileStatus; use App\Jobs\Backup\RestoreDatabase; use App\Jobs\StorageProvider\DeleteFile; use Carbon\Carbon; @@ -75,12 +76,12 @@ public function getPathAttribute(): string public function getStoragePathAttribute(): string { - return '/'.$this->backup->name.'/'.$this->name.'.zip'; + return '/'.$this->backup->database->name.'/'.$this->name.'.zip'; } public function restore(Database $database): void { - $this->status = 'restoring'; + $this->status = BackupFileStatus::RESTORING; $this->restored_to = $database->name; $this->save(); dispatch(new RestoreDatabase($this, $database))->onConnection('ssh'); diff --git a/app/Models/StorageProvider.php b/app/Models/StorageProvider.php index 0282794..6098f5b 100644 --- a/app/Models/StorageProvider.php +++ b/app/Models/StorageProvider.php @@ -2,18 +2,14 @@ namespace App\Models; -use Carbon\Carbon; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\BelongsTo; /** * @property int $user_id + * @property string $profile * @property string $provider - * @property string $label - * @property string $token - * @property string $refresh_token - * @property bool $connected - * @property Carbon $token_expires_at + * @property array $credentials * @property User $user */ class StorageProvider extends AbstractModel @@ -22,18 +18,14 @@ class StorageProvider extends AbstractModel protected $fillable = [ 'user_id', + 'profile', 'provider', - 'label', - 'token', - 'refresh_token', - 'connected', - 'token_expires_at', + 'credentials', ]; protected $casts = [ 'user_id' => 'integer', - 'connected' => 'boolean', - 'token_expires_at' => 'datetime', + 'credentials' => 'encrypted:array', ]; public function user(): BelongsTo diff --git a/app/ServiceHandlers/Database/Mysql.php b/app/ServiceHandlers/Database/Mysql.php index b7bbe8b..dffec82 100755 --- a/app/ServiceHandlers/Database/Mysql.php +++ b/app/ServiceHandlers/Database/Mysql.php @@ -99,7 +99,7 @@ public function runBackup(BackupFile $backupFile): void 'backup-database' ); - // upload to cloud + // upload to storage $upload = $backupFile->backup->storage->provider()->upload( $backupFile->backup->server, $backupFile->path, diff --git a/app/StorageProviders/Dropbox.php b/app/StorageProviders/Dropbox.php index f1c8c33..a2687a7 100644 --- a/app/StorageProviders/Dropbox.php +++ b/app/StorageProviders/Dropbox.php @@ -2,21 +2,25 @@ namespace App\StorageProviders; +use App\Exceptions\BackupFileException; use App\Models\Server; use App\SSHCommands\Storage\DownloadFromDropboxCommand; use App\SSHCommands\Storage\UploadToDropboxCommand; use Illuminate\Support\Facades\Http; -use Laravel\Socialite\Facades\Socialite; -use Symfony\Component\HttpFoundation\RedirectResponse; use Throwable; class Dropbox extends AbstractStorageProvider { - public function connect(): RedirectResponse - { - session()->put('storage_provider_id', $this->storageProvider->id); + protected string $apiUrl = 'https://api.dropboxapi.com/2'; - return Socialite::driver('dropbox')->redirect(); + public function connect(): bool + { + $res = Http::withToken($this->storageProvider->credentials['token']) + ->post($this->apiUrl.'/check/user', [ + 'query' => '' + ]); + + return $res->successful(); } /** @@ -28,15 +32,19 @@ public function upload(Server $server, string $src, string $dest): array new UploadToDropboxCommand( $src, $dest, - $this->storageProvider->token + $this->storageProvider->credentials['token'] ), 'upload-to-dropbox' ); - $data = json_decode($upload); + $data = json_decode($upload, true); + + if (isset($data['error'])) { + throw new BackupFileException("Failed to upload to Dropbox ".$data['error_summary'] ?? ''); + } return [ - 'size' => $data?->size, + 'size' => $data['size'] ?? null, ]; } @@ -49,7 +57,7 @@ public function download(Server $server, string $src, string $dest): void new DownloadFromDropboxCommand( $src, $dest, - $this->storageProvider->token + $this->storageProvider->credentials['token'] ), 'download-from-dropbox' ); @@ -61,11 +69,11 @@ public function delete(array $paths): void foreach ($paths as $path) { $data[] = ['path' => $path]; } - Http::withToken($this->storageProvider->token) + Http::withToken($this->storageProvider->credentials['token']) ->withHeaders([ 'Content-Type:application/json', ]) - ->post('https://api.dropboxapi.com/2/files/delete_batch', [ + ->post($this->apiUrl.'/files/delete_batch', [ 'entries' => $data, ]); } diff --git a/database/factories/BackupFactory.php b/database/factories/BackupFactory.php index 14ed950..d1e678f 100644 --- a/database/factories/BackupFactory.php +++ b/database/factories/BackupFactory.php @@ -2,6 +2,7 @@ namespace Database\Factories; +use App\Enums\BackupStatus; use Illuminate\Database\Eloquent\Factories\Factory; class BackupFactory extends Factory @@ -9,7 +10,10 @@ class BackupFactory extends Factory public function definition(): array { return [ - // + 'type' => 'database', + 'interval' => '0 * * * *', + 'keep_backups' => 10, + 'status' => BackupStatus::RUNNING ]; } } diff --git a/database/factories/StorageProviderFactory.php b/database/factories/StorageProviderFactory.php index 6db29d1..32a28f9 100644 --- a/database/factories/StorageProviderFactory.php +++ b/database/factories/StorageProviderFactory.php @@ -2,6 +2,7 @@ namespace Database\Factories; +use App\Models\User; use Illuminate\Database\Eloquent\Factories\Factory; class StorageProviderFactory extends Factory @@ -9,7 +10,10 @@ class StorageProviderFactory extends Factory public function definition(): array { return [ - // + 'profile' => $this->faker->word(), + 'provider' => $this->faker->randomElement(\App\Enums\StorageProvider::getValues()), + 'credentials' => [], + 'user_id' => User::factory(), ]; } } diff --git a/database/migrations/2023_08_13_095440_update_storage_providers_table.php b/database/migrations/2023_08_13_095440_update_storage_providers_table.php new file mode 100644 index 0000000..be0b2fe --- /dev/null +++ b/database/migrations/2023_08_13_095440_update_storage_providers_table.php @@ -0,0 +1,34 @@ +dropColumn('token'); + $table->dropColumn('refresh_token'); + $table->dropColumn('token_expires_at'); + $table->dropColumn('label'); + $table->dropColumn('connected'); + $table->unsignedBigInteger('user_id')->after('id'); + $table->string('profile')->after('user_id'); + $table->longText('credentials')->nullable()->after('provider'); + }); + } + + public function down(): void + { + Schema::table('storage_providers', function (Blueprint $table) { + $table->string('token'); + $table->string('refresh_token'); + $table->string('token_expires_at'); + $table->string('label'); + $table->dropColumn('user_id'); + $table->dropColumn('profile'); + $table->dropColumn('credentials'); + }); + } +}; diff --git a/database/migrations/2023_08_17_231824_update_backups_table.php b/database/migrations/2023_08_17_231824_update_backups_table.php new file mode 100644 index 0000000..b9bda13 --- /dev/null +++ b/database/migrations/2023_08_17_231824_update_backups_table.php @@ -0,0 +1,21 @@ +dropColumn('name'); + }); + } + + public function down(): void + { + Schema::table('backups', function (Blueprint $table) { + $table->string('name')->nullable(); + }); + } +}; diff --git a/database/migrations/2023_08_25_183201_update_backup_files_table.php b/database/migrations/2023_08_25_183201_update_backup_files_table.php new file mode 100644 index 0000000..26f9ee6 --- /dev/null +++ b/database/migrations/2023_08_25_183201_update_backup_files_table.php @@ -0,0 +1,21 @@ +string('restored_to')->after('status')->nullable(); + }); + } + + public function down(): void + { + Schema::table('backup_files', function (Blueprint $table) { + $table->dropColumn('restored_to'); + }); + } +}; diff --git a/install/install.sh b/install/install.sh index 4ff061b..83e2dcd 100644 --- a/install/install.sh +++ b/install/install.sh @@ -224,6 +224,9 @@ supervisorctl reread supervisorctl update supervisorctl start worker:* +# setup cronjobs +echo "* * * * * cd /home/${V_USERNAME}/${V_DOMAIN} && php artisan schedule:run >> /dev/null 2>&1" | sudo -u ${V_USERNAME} crontab - + # make the update file executable chmod +x /home/${V_USERNAME}/${V_DOMAIN}/update.sh diff --git a/package.json b/package.json index c67950d..60f952a 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "scripts": { "dev": "vite", "build": "vite build", - "queue:listen": "php artisan queue:listen --timeout=600 --queue=default,ssh,ssh-long" + "queue:listen": "php artisan queue:listen --timeout=600 --queue=default,ssh,ssh-long", + "scheduler:run": "php artisan schedule:run" }, "devDependencies": { "@ryangjchandler/alpine-clipboard": "^2.2.0", diff --git a/resources/commands/storage/upload-to-dropbox.sh b/resources/commands/storage/upload-to-dropbox.sh index 0f427a3..82971a7 100644 --- a/resources/commands/storage/upload-to-dropbox.sh +++ b/resources/commands/storage/upload-to-dropbox.sh @@ -1,4 +1,4 @@ -curl --location --request POST 'https://content.dropboxapi.com/2/files/upload' \ +curl -sb --location --request POST 'https://content.dropboxapi.com/2/files/upload' \ --header 'Accept: application/json' \ --header 'Dropbox-API-Arg: {"path":"__dest__"}' \ --header 'Content-Type: text/plain; charset=dropbox-cors-hack' \ diff --git a/resources/views/databases/backups.blade.php b/resources/views/databases/backups.blade.php new file mode 100644 index 0000000..21df878 --- /dev/null +++ b/resources/views/databases/backups.blade.php @@ -0,0 +1,7 @@ + + {{ __("Backup Files") }} + +
+ +
+ diff --git a/resources/views/databases/index.blade.php b/resources/views/databases/index.blade.php index 7495598..1889dd5 100644 --- a/resources/views/databases/index.blade.php +++ b/resources/views/databases/index.blade.php @@ -5,5 +5,7 @@ + + diff --git a/resources/views/layouts/app.blade.php b/resources/views/layouts/app.blade.php index 9fdafe9..00faa21 100644 --- a/resources/views/layouts/app.blade.php +++ b/resources/views/layouts/app.blade.php @@ -58,7 +58,7 @@ @endif @if($server->database()) - + diff --git a/resources/views/layouts/profile.blade.php b/resources/views/layouts/profile.blade.php index 7a81291..323723b 100644 --- a/resources/views/layouts/profile.blade.php +++ b/resources/views/layouts/profile.blade.php @@ -28,6 +28,12 @@ {{ __('Source Controls') }} + + + + + {{ __('Storage Providers') }} + diff --git a/resources/views/livewire/broadcast.blade.php b/resources/views/livewire/broadcast.blade.php index b86acdd..aa2ea62 100644 --- a/resources/views/livewire/broadcast.blade.php +++ b/resources/views/livewire/broadcast.blade.php @@ -1 +1 @@ -
+
diff --git a/resources/views/livewire/databases/database-backup-files.blade.php b/resources/views/livewire/databases/database-backup-files.blade.php new file mode 100644 index 0000000..c26db0d --- /dev/null +++ b/resources/views/livewire/databases/database-backup-files.blade.php @@ -0,0 +1,74 @@ +
+ + {{ __("Backup Files") }} + {{ __("Here you can see your backup files") }} + +
+ + {{ __('Back to Databases') }} + + + {{ __("Backup Now") }} + +
+ + + @if(count($files) > 0) + + + {{ __("Name") }} + {{ __("Created") }} + {{ __("Size") }} + {{ __("Status") }} + {{ __("Restored") }} + {{ __("Restored To") }} + + + @foreach($files as $file) + + {{ $file->name }} + + + + {{ $file->size }} + +
+ @include('livewire.databases.partials.backup-file-status', ['status' => $file->status]) +
+ + + @if($file->restored_at) + + @else + - + @endif + + + @if($file->restored_to) + {{ $file->restored_to }} + @else + - + @endif + + + @if(in_array($file->status, [\App\Enums\BackupFileStatus::CREATED, \App\Enums\BackupFileStatus::RESTORED, \App\Enums\BackupFileStatus::RESTORE_FAILED])) + + Restore + + @endif + + Delete + + + + @endforeach + +
+ {{ $files->withQueryString()->links() }} +
+ @include('livewire.databases.partials.restore-backup-modal', ['databases' => $server->databases]) + @include('livewire.databases.partials.delete-backup-file-modal') + @else + {{ __("You don't have any backups yet") }} + @endif +
diff --git a/resources/views/livewire/databases/database-backups.blade.php b/resources/views/livewire/databases/database-backups.blade.php new file mode 100644 index 0000000..89c97de --- /dev/null +++ b/resources/views/livewire/databases/database-backups.blade.php @@ -0,0 +1,49 @@ +
+ + {{ __("Backups") }} + {{ __("You can backup your databases into external storages") }} + +
+ + {{ __('Create Backup') }} + + + @include('livewire.databases.partials.create-backup-modal') +
+ + + @if(count($backups) > 0) + + + {{ __("Database") }} + {{ __("Created") }} + {{ __("Status") }} + + + @foreach($backups as $backup) + + {{ $backup->database->name }} + + + + +
+ @include('livewire.databases.partials.backup-status', ['status' => $backup->status]) +
+ + + + Files + + + Delete + + + + @endforeach + + @include('livewire.databases.partials.delete-backup-modal') + @else + {{ __("You don't have any backups yet") }} + @endif +
diff --git a/resources/views/livewire/databases/database-list.blade.php b/resources/views/livewire/databases/database-list.blade.php index 554c1a3..ca5d32e 100644 --- a/resources/views/livewire/databases/database-list.blade.php +++ b/resources/views/livewire/databases/database-list.blade.php @@ -15,10 +15,10 @@ @if(count($databases) > 0) - {{ __("Name") }} - {{ __("Created") }} - {{ __("Status") }} - + {{ __("Name") }} + {{ __("Created") }} + {{ __("Status") }} + @foreach($databases as $database) diff --git a/resources/views/livewire/databases/database-user-list.blade.php b/resources/views/livewire/databases/database-user-list.blade.php index e465bc6..3c53ad1 100644 --- a/resources/views/livewire/databases/database-user-list.blade.php +++ b/resources/views/livewire/databases/database-user-list.blade.php @@ -15,11 +15,11 @@ @if(count($databaseUsers) > 0) - {{ __("Username") }} - {{ __("Created") }} - {{ __("Linked Databases") }} - {{ __("Status") }} - + {{ __("Username") }} + {{ __("Created") }} + {{ __("Linked Databases") }} + {{ __("Status") }} + @foreach($databaseUsers as $databaseUser) diff --git a/resources/views/livewire/databases/partials/backup-file-status.blade.php b/resources/views/livewire/databases/partials/backup-file-status.blade.php new file mode 100644 index 0000000..7909681 --- /dev/null +++ b/resources/views/livewire/databases/partials/backup-file-status.blade.php @@ -0,0 +1,21 @@ +@if($status == \App\Enums\BackupFileStatus::CREATED) + {{ $status }} +@endif +@if($status == \App\Enums\BackupFileStatus::CREATING) + {{ $status }} +@endif +@if($status == \App\Enums\BackupFileStatus::FAILED) + {{ $status }} +@endif +@if($status == \App\Enums\BackupFileStatus::DELETING) + {{ $status }} +@endif +@if($status == \App\Enums\BackupFileStatus::RESTORING) + {{ $status }} +@endif +@if($status == \App\Enums\BackupFileStatus::RESTORED) + {{ $status }} +@endif +@if($status == \App\Enums\BackupFileStatus::RESTORE_FAILED) + {{ $status }} +@endif diff --git a/resources/views/livewire/databases/partials/backup-status.blade.php b/resources/views/livewire/databases/partials/backup-status.blade.php new file mode 100644 index 0000000..a80ad4e --- /dev/null +++ b/resources/views/livewire/databases/partials/backup-status.blade.php @@ -0,0 +1,9 @@ +@if($status == \App\Enums\BackupStatus::RUNNING) + {{ $status }} +@endif +@if($status == \App\Enums\BackupStatus::DELETING) + {{ $status }} +@endif +@if($status == \App\Enums\BackupStatus::FAILED) + {{ $status }} +@endif diff --git a/resources/views/livewire/databases/partials/create-backup-modal.blade.php b/resources/views/livewire/databases/partials/create-backup-modal.blade.php new file mode 100644 index 0000000..74b6791 --- /dev/null +++ b/resources/views/livewire/databases/partials/create-backup-modal.blade.php @@ -0,0 +1,78 @@ + +
+

+ {{ __('Create Backup') }} +

+ +
+ + + + @foreach($databases as $db) + + @endforeach + + @error('database') + + @enderror +
+ +
+ + + + @foreach(auth()->user()->storageProviders as $st) + + @endforeach + + @error('storage') + + @enderror +
+ + +
+ + + + + + + + + + @error('interval') + + @enderror +
+ + + @if($interval === 'custom') +
+ + + @error('custom') + + @enderror +
+ @endif + +
+ + + @error('keep') + + @enderror +
+ +
+ + {{ __('Cancel') }} + + + + {{ __('Create') }} + +
+
+ diff --git a/resources/views/livewire/databases/partials/delete-backup-file-modal.blade.php b/resources/views/livewire/databases/partials/delete-backup-file-modal.blade.php new file mode 100644 index 0000000..ca638ed --- /dev/null +++ b/resources/views/livewire/databases/partials/delete-backup-file-modal.blade.php @@ -0,0 +1,6 @@ + diff --git a/resources/views/livewire/databases/partials/delete-backup-modal.blade.php b/resources/views/livewire/databases/partials/delete-backup-modal.blade.php new file mode 100644 index 0000000..c48fc14 --- /dev/null +++ b/resources/views/livewire/databases/partials/delete-backup-modal.blade.php @@ -0,0 +1,6 @@ + diff --git a/resources/views/livewire/databases/partials/restore-backup-modal.blade.php b/resources/views/livewire/databases/partials/restore-backup-modal.blade.php new file mode 100644 index 0000000..952a76e --- /dev/null +++ b/resources/views/livewire/databases/partials/restore-backup-modal.blade.php @@ -0,0 +1,30 @@ + +
+

+ {{ __('Restore Backup') }} +

+ +
+ + + + @foreach($databases as $db) + + @endforeach + + @error('restoreDatabaseId') + + @enderror +
+ +
+ + {{ __('Cancel') }} + + + + {{ __('Restore') }} + +
+
+ diff --git a/resources/views/livewire/storage-providers/connect-provider.blade.php b/resources/views/livewire/storage-providers/connect-provider.blade.php new file mode 100644 index 0000000..223e3ab --- /dev/null +++ b/resources/views/livewire/storage-providers/connect-provider.blade.php @@ -0,0 +1,54 @@ +
+ + {{ __('Connect') }} + + + +
+

+ {{ __('Connect to a Storage Provider') }} +

+ +
+ + + + @foreach(config('core.storage_providers') as $p) + @if($p !== 'custom') + + @endif + @endforeach + + @error('provider') + + @enderror +
+ +
+ + + @error('name') + + @enderror +
+ +
+ + + @error('token') + + @enderror +
+ +
+ + {{ __('Cancel') }} + + + + {{ __('Connect') }} + +
+
+ +
diff --git a/resources/views/livewire/storage-providers/providers-list.blade.php b/resources/views/livewire/storage-providers/providers-list.blade.php new file mode 100644 index 0000000..58a53ba --- /dev/null +++ b/resources/views/livewire/storage-providers/providers-list.blade.php @@ -0,0 +1,45 @@ +
+ + Storage Providers + You can connect to your storage providers + + + + +
+ @if(count($providers) > 0) + @foreach($providers as $provider) + +
+ +
+
+ {{ $provider->profile }} + + + +
+
+
+ + Delete + +
+
+ + @endforeach + + @else + +
+ {{ __("You haven't connected to any storage providers yet!") }} +
+ + @endif +
+
diff --git a/resources/views/server-providers/index.blade.php b/resources/views/server-providers/index.blade.php index f089020..f79d1fa 100644 --- a/resources/views/server-providers/index.blade.php +++ b/resources/views/server-providers/index.blade.php @@ -1,5 +1,5 @@ - {{ __("Server Providers") }} + {{ __("Storage Providers") }} diff --git a/resources/views/storage-providers/index.blade.php b/resources/views/storage-providers/index.blade.php new file mode 100644 index 0000000..f4b5a60 --- /dev/null +++ b/resources/views/storage-providers/index.blade.php @@ -0,0 +1,5 @@ + + {{ __("Storage Providers") }} + + + diff --git a/routes/web.php b/routes/web.php index 43ea29b..b848e49 100644 --- a/routes/web.php +++ b/routes/web.php @@ -21,6 +21,7 @@ Route::view('/profile', 'profile.index')->name('profile'); Route::view('/server-providers', 'server-providers.index')->name('server-providers'); Route::view('/source-controls', 'source-controls.index')->name('source-controls'); + Route::view('/storage-providers', 'storage-providers.index')->name('storage-providers'); Route::view('/notification-channels', 'notification-channels.index')->name('notification-channels'); Route::view('/ssh-keys', 'ssh-keys.index')->name('ssh-keys'); }); @@ -32,6 +33,9 @@ Route::get('/{server}/settings', [ServerSettingController::class, 'index'])->name('servers.settings'); Route::middleware('server-is-ready')->group(function () { Route::get('/{server}/databases', [DatabaseController::class, 'index'])->name('servers.databases'); + Route::get('/{server}/databases/backups/{backup}', [DatabaseController::class, 'backups'])->name( + 'servers.databases.backups' + ); Route::prefix('/{server}/sites')->group(function () { Route::get('/', [SiteController::class, 'index'])->name('servers.sites'); Route::get('/create', [SiteController::class, 'create'])->name('servers.sites.create'); diff --git a/tests/Feature/Http/DatabaseBackupTest.php b/tests/Feature/Http/DatabaseBackupTest.php new file mode 100644 index 0000000..fa21475 --- /dev/null +++ b/tests/Feature/Http/DatabaseBackupTest.php @@ -0,0 +1,105 @@ +actingAs($this->user); + + Bus::fake(); + + SSH::fake()->outputShouldBe('test'); + + $database = Database::factory()->create([ + 'server_id' => $this->server, + ]); + + $storage = StorageProvider::factory()->create([ + 'user_id' => $this->user->id, + 'provider' => \App\Enums\StorageProvider::DROPBOX + ]); + + Livewire::test(DatabaseBackups::class, ['server' => $this->server]) + ->set('database', $database->id) + ->set('storage', $storage->id) + ->set('interval', '0 * * * *') + ->set('keep', '10') + ->call('create') + ->assertSuccessful(); + + Bus::assertDispatched(RunBackup::class); + + $this->assertDatabaseHas('backups', [ + 'status' => BackupStatus::RUNNING, + ]); + } + + public function test_see_backups_list(): void + { + $this->actingAs($this->user); + + $database = Database::factory()->create([ + 'server_id' => $this->server, + ]); + + $storage = StorageProvider::factory()->create([ + 'user_id' => $this->user->id, + 'provider' => \App\Enums\StorageProvider::DROPBOX + ]); + + $backup = Backup::factory()->create([ + 'server_id' => $this->server->id, + 'database_id' => $database->id, + 'storage_id' => $storage->id, + ]); + + Livewire::test(DatabaseBackups::class, ['server' => $this->server]) + ->assertSee([ + $backup->database->name, + ]); + } + + public function test_delete_database(): void + { + $this->actingAs($this->user); + + $database = Database::factory()->create([ + 'server_id' => $this->server, + ]); + + $storage = StorageProvider::factory()->create([ + 'user_id' => $this->user->id, + 'provider' => \App\Enums\StorageProvider::DROPBOX + ]); + + $backup = Backup::factory()->create([ + 'server_id' => $this->server->id, + 'database_id' => $database->id, + 'storage_id' => $storage->id, + ]); + + Livewire::test(DatabaseBackups::class, ['server' => $this->server]) + ->set('deleteId', $backup->id) + ->call('delete'); + + $this->assertDatabaseMissing('backups', [ + 'id' => $backup->id, + ]); + } +} diff --git a/tests/Feature/Http/StorageProvidersTest.php b/tests/Feature/Http/StorageProvidersTest.php new file mode 100644 index 0000000..36399cf --- /dev/null +++ b/tests/Feature/Http/StorageProvidersTest.php @@ -0,0 +1,68 @@ +actingAs($this->user); + + Http::fake(); + + Livewire::test(ConnectProvider::class) + ->set('provider', StorageProvider::DROPBOX) + ->set('name', 'profile') + ->set('token', 'token') + ->call('connect') + ->assertSuccessful(); + + $this->assertDatabaseHas('storage_providers', [ + 'provider' => StorageProvider::DROPBOX, + 'profile' => 'profile', + ]); + } + + public function test_see_providers_list(): void + { + $this->actingAs($this->user); + + $provider = \App\Models\StorageProvider::factory()->create([ + 'user_id' => $this->user->id, + 'provider' => StorageProvider::DROPBOX + ]); + + Livewire::test(ProvidersList::class) + ->assertSee([ + $provider->profile, + ]); + } + + public function test_delete_provider(): void + { + $this->actingAs($this->user); + + $provider = \App\Models\StorageProvider::factory()->create([ + 'user_id' => $this->user->id, + ]); + + Livewire::test(ProvidersList::class) + ->set('deleteId', $provider->id) + ->call('delete') + ->assertSuccessful(); + + $this->assertDatabaseMissing('storage_providers', [ + 'id' => $provider->id, + ]); + } +}