support S3 compatible storage providers (#401)

This commit is contained in:
Saeed Vaziry 2024-12-22 23:06:36 +01:00 committed by GitHub
parent db81583884
commit ea3d64607a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 127 additions and 576 deletions

View File

@ -11,6 +11,4 @@ final class StorageProvider
const LOCAL = 'local';
const S3 = 's3';
const WASABI = 'wasabi';
}

View File

@ -3,36 +3,31 @@
namespace App\SSH\Storage;
use App\Exceptions\SSHCommandError;
use App\Models\Server;
use App\Models\StorageProvider;
use App\Exceptions\SSHError;
use App\SSH\HasS3Storage;
use App\SSH\HasScripts;
use Illuminate\Support\Facades\Log;
class S3 extends S3AbstractStorage
class S3 extends AbstractStorage
{
use HasS3Storage, HasScripts;
public function __construct(Server $server, StorageProvider $storageProvider)
{
parent::__construct($server, $storageProvider);
$this->setBucketRegion($this->storageProvider->credentials['region']);
$this->setApiUrl();
}
/**
* @throws SSHCommandError
* @throws SSHError
*/
public function upload(string $src, string $dest): array
{
/** @var \App\StorageProviders\S3 $provider */
$provider = $this->storageProvider->provider();
$uploadCommand = $this->getScript('s3/upload.sh', [
'src' => $src,
'bucket' => $this->storageProvider->credentials['bucket'],
'dest' => $this->prepareS3Path($this->storageProvider->credentials['path'].'/'.$dest),
'key' => $this->storageProvider->credentials['key'],
'secret' => $this->storageProvider->credentials['secret'],
'region' => $this->getBucketRegion(),
'endpoint' => $this->getApiUrl(),
'region' => $this->storageProvider->credentials['region'],
'endpoint' => $provider->getApiUrl(),
]);
$upload = $this->server->ssh()->exec($uploadCommand, 'upload-to-s3');
@ -49,20 +44,25 @@ public function upload(string $src, string $dest): array
}
/**
* @throws SSHCommandError
* @throws SSHError
*/
public function download(string $src, string $dest): void
{
/** @var \App\StorageProviders\S3 $provider */
$provider = $this->storageProvider->provider();
$downloadCommand = $this->getScript('s3/download.sh', [
'src' => $this->prepareS3Path($this->storageProvider->credentials['path'].'/'.$src),
'dest' => $dest,
'bucket' => $this->storageProvider->credentials['bucket'],
'key' => $this->storageProvider->credentials['key'],
'secret' => $this->storageProvider->credentials['secret'],
'region' => $this->getBucketRegion(),
'endpoint' => $this->getApiUrl(),
'region' => $this->storageProvider->credentials['region'],
'endpoint' => $provider->getApiUrl(),
]);
Log::info('Downloading from S3', ['command' => $downloadCommand]);
$download = $this->server->ssh()->exec($downloadCommand, 'download-from-s3');
if (! str_contains($download, 'Download successful')) {

View File

@ -1,32 +0,0 @@
<?php
namespace App\SSH\Storage;
abstract class S3AbstractStorage extends AbstractStorage
{
protected ?string $apiUrl = null;
protected ?string $bucketRegion = null;
public function getApiUrl(): string
{
return $this->apiUrl;
}
public function setApiUrl(?string $region = null): void
{
$this->bucketRegion = $region ?? $this->bucketRegion;
$this->apiUrl = "https://s3.{$this->bucketRegion}.amazonaws.com";
}
// Getter and Setter for $bucketRegion
public function getBucketRegion(): string
{
return $this->bucketRegion;
}
public function setBucketRegion(string $region): void
{
$this->bucketRegion = $region;
}
}

View File

@ -1,84 +0,0 @@
<?php
namespace App\SSH\Storage;
use App\Exceptions\SSHCommandError;
use App\Models\Server;
use App\Models\StorageProvider;
use App\SSH\HasS3Storage;
use App\SSH\HasScripts;
use Illuminate\Support\Facades\Log;
class Wasabi extends S3AbstractStorage
{
use HasS3Storage, HasScripts;
public function __construct(Server $server, StorageProvider $storageProvider)
{
parent::__construct($server, $storageProvider);
$this->setBucketRegion($this->storageProvider->credentials['region']);
$this->setApiUrl();
}
/**
* @throws SSHCommandError
*/
public function upload(string $src, string $dest): array
{
$uploadCommand = $this->getScript('wasabi/upload.sh', [
'src' => $src,
'bucket' => $this->storageProvider->credentials['bucket'],
'dest' => $this->prepareS3Path($this->storageProvider->credentials['path'].'/'.$dest),
'key' => $this->storageProvider->credentials['key'],
'secret' => $this->storageProvider->credentials['secret'],
'region' => $this->storageProvider->credentials['region'],
'endpoint' => $this->getApiUrl(),
]);
$upload = $this->server->ssh()->exec($uploadCommand, 'upload-to-wasabi');
if (str_contains($upload, 'Error') || ! str_contains($upload, 'upload:')) {
Log::error('Failed to upload to wasabi', ['output' => $upload]);
throw new SSHCommandError('Failed to upload to wasabi: '.$upload);
}
return [
'size' => null, // You can parse the size from the output if needed
];
}
/**
* @throws SSHCommandError
*/
public function download(string $src, string $dest): void
{
$downloadCommand = $this->getScript('wasabi/download.sh', [
'src' => $this->prepareS3Path($this->storageProvider->credentials['path'].'/'.$src),
'dest' => $dest,
'bucket' => $this->storageProvider->credentials['bucket'],
'key' => $this->storageProvider->credentials['key'],
'secret' => $this->storageProvider->credentials['secret'],
'region' => $this->storageProvider->credentials['region'],
'endpoint' => $this->getApiUrl(),
]);
$download = $this->server->ssh()->exec($downloadCommand, 'download-from-wasabi');
if (! str_contains($download, 'Download successful')) {
Log::error('Failed to download from wasabi', ['output' => $download]);
throw new SSHCommandError('Failed to download from wasabi: '.$download);
}
}
/**
* @TODO Implement delete method
*/
public function delete(string $path): void {}
public function setApiUrl(?string $region = null): void
{
$this->bucketRegion = $region ?? $this->bucketRegion;
$this->apiUrl = "https://{$this->storageProvider->credentials['bucket']}.s3.{$this->getBucketRegion()}.wasabisys.com";
}
}

View File

@ -1,31 +0,0 @@
#!/bin/bash
# Configure AWS CLI with provided credentials
/usr/local/bin/aws configure set aws_access_key_id "__key__"
/usr/local/bin/aws configure set aws_secret_access_key "__secret__"
# Use the provided endpoint in the correct format
ENDPOINT="__endpoint__"
BUCKET="__bucket__"
REGION="__region__"
# Ensure that DEST does not have a trailing slash
SRC="__src__"
DEST="__dest__"
# Download the file from S3
echo "Downloading s3://__bucket____src__ to __dest__"
download_output=$(/usr/local/bin/aws s3 cp "s3://$BUCKET/$SRC" "$DEST" --endpoint-url="$ENDPOINT" --region "$REGION" 2>&1)
download_exit_code=$?
# Log output and exit code
echo "Download command output: $download_output"
echo "Download command exit code: $download_exit_code"
# Check if the download was successful
if [ $download_exit_code -eq 0 ]; then
echo "Download successful"
else
echo "Download failed"
exit 1
fi

View File

@ -1,59 +0,0 @@
#!/bin/bash
# Check if AWS CLI is installed
if ! command -v aws &> /dev/null
then
echo "AWS CLI is not installed. Installing..."
# Detect system architecture
ARCH=$(uname -m)
if [ "$ARCH" == "x86_64" ]; then
CLI_URL="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"
elif [ "$ARCH" == "aarch64" ]; then
CLI_URL="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip"
else
echo "Unsupported architecture: $ARCH"
exit 1
fi
# Download and install AWS CLI
sudo curl "$CLI_URL" -o "awscliv2.zip"
sudo unzip awscliv2.zip
sudo ./aws/install --update
sudo rm -rf awscliv2.zip aws
echo "AWS CLI installation completed."
else
echo "AWS CLI is already installed."
aws --version
fi
# Configure AWS CLI with provided credentials
/usr/local/bin/aws configure set aws_access_key_id "__key__"
/usr/local/bin/aws configure set aws_secret_access_key "__secret__"
# Use the provided endpoint in the correct format
ENDPOINT="__endpoint__"
BUCKET="__bucket__"
REGION="__region__"
# Ensure that DEST does not have a trailing slash
SRC="__src__"
DEST="__dest__"
# Upload the file
echo "Uploading __src__ to s3://$BUCKET/$DEST"
upload_output=$(/usr/local/bin/aws s3 cp "$SRC" "s3://$BUCKET/$DEST" --endpoint-url="$ENDPOINT" --region "$REGION" 2>&1)
upload_exit_code=$?
# Log output and exit code
echo "Upload command output: $upload_output"
echo "Upload command exit code: $upload_exit_code"
# Check if the upload was successful
if [ $upload_exit_code -eq 0 ]; then
echo "Upload successful"
else
echo "Upload failed"
exit 1
fi

View File

@ -3,46 +3,89 @@
namespace App\StorageProviders;
use App\Models\Server;
use App\Models\StorageProvider;
use App\SSH\Storage\S3 as S3Storage;
use App\SSH\Storage\Storage;
use Aws\S3\Exception\S3Exception;
use Aws\S3\S3Client;
use Illuminate\Support\Facades\Log;
class S3 extends S3AbstractStorageProvider
class S3 extends AbstractStorageProvider
{
protected StorageProvider $storageProvider;
protected ?S3Client $client = null;
protected array $clientConfig = [];
public function getApiUrl(): string
{
if (isset($this->storageProvider->credentials['api_url']) && $this->storageProvider->credentials['api_url']) {
return $this->storageProvider->credentials['api_url'];
}
$region = $this->storageProvider->credentials['region'];
return "https://s3.{$region}.amazonaws.com";
}
public function getClient(): S3Client
{
return new S3Client($this->clientConfig);
}
/**
* Build the configuration array for the S3 client.
* This method can be overridden by child classes to modify the configuration.
*/
public function buildClientConfig(): array
{
$this->clientConfig = [
'credentials' => [
'key' => $this->storageProvider->credentials['key'],
'secret' => $this->storageProvider->credentials['secret'],
],
'region' => $this->storageProvider->credentials['region'],
'version' => 'latest',
'endpoint' => $this->getApiUrl(),
];
return $this->clientConfig;
}
public function validationRules(): array
{
return [
'api_url' => 'nullable',
'key' => 'required',
'secret' => 'required',
'region' => 'required',
'bucket' => 'required',
'path' => 'required',
'path' => 'nullable',
];
}
public function credentialData(array $input): array
{
return [
'api_url' => $input['api_url'] ?? '',
'key' => $input['key'],
'secret' => $input['secret'],
'region' => $input['region'],
'bucket' => $input['bucket'],
'path' => $input['path'],
'path' => $input['path'] ?? '',
];
}
public function connect(): bool
{
try {
$this->setBucketRegion($this->storageProvider->credentials['region']);
$this->setApiUrl();
$this->buildClientConfig();
$this->getClient()->listBuckets();
return true;
} catch (S3Exception $e) {
Log::error('Failed to connect to S3', ['exception' => $e]);
Log::error('Failed to connect to the provider', ['exception' => $e]);
return false;
}

View File

@ -1,74 +0,0 @@
<?php
namespace App\StorageProviders;
use App\Models\StorageProvider;
use Aws\S3\S3Client;
abstract class S3AbstractStorageProvider extends AbstractStorageProvider implements S3ClientInterface, S3StorageInterface
{
protected ?string $apiUrl = null;
protected ?string $bucketRegion = null;
protected ?S3Client $client = null;
protected StorageProvider $storageProvider;
protected array $clientConfig = [];
public function getApiUrl(): string
{
return $this->apiUrl;
}
public function setApiUrl(?string $region = null): void
{
$this->bucketRegion = $region ?? $this->bucketRegion;
$this->apiUrl = "https://s3.{$this->bucketRegion}.amazonaws.com";
}
public function getBucketRegion(): string
{
return $this->bucketRegion;
}
public function setBucketRegion(string $region): void
{
$this->bucketRegion = $region;
}
public function getClient(): S3Client
{
return new S3Client($this->clientConfig);
}
/**
* Build the configuration array for the S3 client.
* This method can be overridden by child classes to modify the configuration.
*/
public function buildClientConfig(): array
{
$this->clientConfig = [
'credentials' => [
'key' => $this->storageProvider->credentials['key'],
'secret' => $this->storageProvider->credentials['secret'],
],
'region' => $this->getBucketRegion(),
'version' => 'latest',
'endpoint' => $this->getApiUrl(),
];
return $this->clientConfig;
}
/**
* Set or update a configuration parameter for the S3 client.
*/
public function setConfigParam(array $param): void
{
foreach ($param as $key => $value) {
$this->clientConfig[$key] = $value;
}
}
}

View File

@ -1,10 +0,0 @@
<?php
namespace App\StorageProviders;
use Aws\S3\S3Client;
interface S3ClientInterface
{
public function getClient(): S3Client;
}

View File

@ -1,14 +0,0 @@
<?php
namespace App\StorageProviders;
interface S3StorageInterface
{
public function getApiUrl(): string;
public function setApiUrl(?string $region = null): void;
public function getBucketRegion(): string;
public function setBucketRegion(string $region): void;
}

View File

@ -1,85 +0,0 @@
<?php
namespace App\StorageProviders;
use App\Models\Server;
use App\SSH\Storage\Storage;
use App\SSH\Storage\Wasabi as WasabiStorage;
use Aws\S3\Exception\S3Exception;
use Illuminate\Support\Facades\Log;
class Wasabi extends S3AbstractStorageProvider
{
private const DEFAULT_REGION = 'us-east-1';
public function validationRules(): array
{
return [
'key' => 'required',
'secret' => 'required',
'region' => 'required',
'bucket' => 'required',
'path' => 'required',
];
}
public function credentialData(array $input): array
{
return [
'key' => $input['key'],
'secret' => $input['secret'],
'region' => $input['region'],
'bucket' => $input['bucket'],
'path' => $input['path'],
];
}
public function connect(): bool
{
try {
$this->setBucketRegion(self::DEFAULT_REGION);
$this->setApiUrl();
$this->buildClientConfig();
$this->getClient()->listBuckets();
return true;
} catch (S3Exception $e) {
Log::error('Failed to connect to S3', ['exception' => $e]);
return false;
}
}
/**
* Build the configuration array for the S3 client.
* This method can be overridden by child classes to modify the configuration.
*/
public function buildClientConfig(): array
{
$this->clientConfig = [
'credentials' => [
'key' => $this->storageProvider->credentials['key'],
'secret' => $this->storageProvider->credentials['secret'],
],
'region' => $this->getBucketRegion(),
'version' => 'latest',
'endpoint' => $this->getApiUrl(),
'use_path_style_endpoint' => true,
];
return $this->clientConfig;
}
public function ssh(Server $server): Storage
{
return new WasabiStorage($server, $this->storageProvider);
}
public function setApiUrl(?string $region = null): void
{
$this->bucketRegion = $region ?? $this->bucketRegion;
$this->apiUrl = "https://s3.{$this->bucketRegion}.wasabisys.com";
}
public function delete(array $paths): void {}
}

View File

@ -55,10 +55,14 @@ public static function form(): array
->visible(fn ($get) => $get('provider') == StorageProvider::FTP)
->rules(fn ($get) => CreateStorageProvider::rules($get())['passive']),
]),
TextInput::make('api_url')
->label('API URL')
->visible(fn ($get) => $get('provider') == StorageProvider::S3)
->rules(fn ($get) => CreateStorageProvider::rules($get())['api_url'])
->helperText('Required if you are using an S3 compatible provider like Cloudflare R2'),
TextInput::make('path')
->visible(fn ($get) => in_array($get('provider'), [
StorageProvider::S3,
StorageProvider::WASABI,
StorageProvider::FTP,
StorageProvider::LOCAL,
]))
@ -70,16 +74,9 @@ public static function form(): array
};
}),
Grid::make()
->visible(fn ($get) => in_array($get('provider'), [
StorageProvider::S3,
StorageProvider::WASABI,
]))
->visible(fn ($get) => $get('provider') == StorageProvider::S3)
->schema([
TextInput::make('key')
->visible(fn ($get) => in_array($get('provider'), [
StorageProvider::S3,
StorageProvider::WASABI,
]))
->rules(fn ($get) => CreateStorageProvider::rules($get())['key'])
->helperText(function ($get) {
return match ($get('provider')) {
@ -88,31 +85,14 @@ public static function form(): array
text: 'How to generate?',
external: true
),
StorageProvider::WASABI => new Link(
href: 'https://docs.wasabi.com/docs/creating-a-user-account-and-access-key',
text: 'How to generate?',
external: true
),
default => '',
};
}),
TextInput::make('secret')
->visible(fn ($get) => in_array($get('provider'), [
StorageProvider::S3,
StorageProvider::WASABI,
]))
->rules(fn ($get) => CreateStorageProvider::rules($get())['secret']),
TextInput::make('region')
->visible(fn ($get) => in_array($get('provider'), [
StorageProvider::S3,
StorageProvider::WASABI,
]))
->rules(fn ($get) => CreateStorageProvider::rules($get())['region']),
TextInput::make('bucket')
->visible(fn ($get) => in_array($get('provider'), [
StorageProvider::S3,
StorageProvider::WASABI,
]))
->rules(fn ($get) => CreateStorageProvider::rules($get())['bucket']),
]),
Checkbox::make('global')

View File

@ -489,15 +489,12 @@
\App\Enums\StorageProvider::FTP,
\App\Enums\StorageProvider::LOCAL,
\App\Enums\StorageProvider::S3,
\App\Enums\StorageProvider::WASABI,
],
'storage_providers_class' => [
\App\Enums\StorageProvider::DROPBOX => \App\StorageProviders\Dropbox::class,
\App\Enums\StorageProvider::FTP => \App\StorageProviders\FTP::class,
\App\Enums\StorageProvider::LOCAL => \App\StorageProviders\Local::class,
\App\Enums\StorageProvider::S3 => \App\StorageProviders\S3::class,
\App\Enums\StorageProvider::WASABI => \App\StorageProviders\Wasabi::class,
],
'ssl_types' => [

View File

@ -0,0 +1,28 @@
<?php
use App\Enums\StorageProvider;
use Illuminate\Database\Migrations\Migration;
return new class extends Migration
{
public function up(): void
{
$wasabiProviders = \App\Models\StorageProvider::query()
->where('provider', 'wasabi')
->get();
/** @var \App\Models\StorageProvider $provider */
foreach ($wasabiProviders as $provider) {
$provider->provider = StorageProvider::S3;
$credentials = $provider->credentials;
$credentials['api_url'] = "https://{$credentials['bucket']}.s3.{$credentials['region']}.wasabisys.com";
$provider->credentials = $credentials;
$provider->save();
}
}
public function down(): void
{
//
}
};

View File

@ -14,9 +14,13 @@ class DatabaseSeeder extends Seeder
*/
public function run(): void
{
$this->call([
$seeders = [
ProjectsSeeder::class,
UsersSeeder::class,
];
if (config('app.demo')) {
$seeders = array_merge($seeders, [
TagsSeeder::class,
ServerProvidersSeeder::class,
StorageProvidersSeeder::class,
@ -31,4 +35,7 @@ public function run(): void
ServerLogsSeeder::class,
]);
}
$this->call($seeders);
}
}

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 6.6 KiB

After

Width:  |  Height:  |  Size: 742 B

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 51 KiB

View File

@ -52,6 +52,7 @@ public function test_see_providers_list(): void
{
$this->actingAs($this->user);
/** @var StorageProviderModel $provider */
$provider = StorageProviderModel::factory()->create([
'user_id' => $this->user->id,
'provider' => StorageProvider::DROPBOX,
@ -61,6 +62,7 @@ public function test_see_providers_list(): void
->assertSuccessful()
->assertSee($provider->profile);
/** @var StorageProviderModel $provider */
$provider = StorageProviderModel::factory()->create([
'user_id' => $this->user->id,
'provider' => StorageProvider::S3,
@ -70,11 +72,6 @@ public function test_see_providers_list(): void
->assertSuccessful()
->assertSee($provider->profile);
$provider = StorageProviderModel::factory()->create([
'user_id' => $this->user->id,
'provider' => StorageProvider::WASABI,
]);
$this->get(Index::getUrl())
->assertSuccessful()
->assertSee($provider->profile);

View File

@ -20,6 +20,7 @@ public function test_s3_connect_successful()
$storageProvider = StorageProviderModel::factory()->create([
'provider' => StorageProvider::S3,
'credentials' => [
'api_url' => 'https://fake-bucket.s3.us-east-1.s3-compatible.com',
'key' => 'fake-key',
'secret' => 'fake-secret',
'region' => 'us-east-1',

View File

@ -1,104 +0,0 @@
<?php
namespace Tests\Unit\StorageProviders;
use App\Enums\StorageProvider;
use App\Models\StorageProvider as StorageProviderModel;
use App\StorageProviders\S3;
use App\StorageProviders\Wasabi;
use Aws\Command;
use Aws\S3\Exception\S3Exception;
use Aws\S3\S3Client;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Tests\TestCase;
class WasabiTest extends TestCase
{
use RefreshDatabase;
public function test_wasabi_connect_successful()
{
$storageProvider = StorageProviderModel::factory()->create([
'provider' => StorageProvider::WASABI,
'credentials' => [
'key' => 'fake-key',
'secret' => 'fake-secret',
'region' => 'us-east-1',
'bucket' => 'fake-bucket',
'path' => '/',
],
]);
// Mock the S3Client (Wasabi uses S3-compatible API)
$s3ClientMock = $this->getMockBuilder(S3Client::class)
->disableOriginalConstructor()
->onlyMethods(['getCommand', 'execute'])
->getMock();
// Mock the getCommand method
$s3ClientMock->expects($this->once())
->method('getCommand')
->with('listBuckets')
->willReturn(new Command('listBuckets'));
// Mock the execute method
$s3ClientMock->expects($this->once())
->method('execute')
->willReturn(['Buckets' => []]);
// Mock the Wasabi class to return the mocked S3Client
$wasabi = $this->getMockBuilder(Wasabi::class)
->setConstructorArgs([$storageProvider])
->onlyMethods(['getClient'])
->getMock();
$wasabi->expects($this->once())
->method('getClient')
->willReturn($s3ClientMock);
$this->assertTrue($wasabi->connect());
}
public function test_wasabi_connect_failure()
{
$storageProvider = StorageProviderModel::factory()->create([
'provider' => StorageProvider::WASABI,
'credentials' => [
'key' => 'fake-key',
'secret' => 'fake-secret',
'region' => 'us-east-1',
'bucket' => 'fake-bucket',
'path' => '/',
],
]);
// Mock the S3Client (Wasabi uses S3-compatible API)
$s3ClientMock = $this->getMockBuilder(S3Client::class)
->disableOriginalConstructor()
->onlyMethods(['getCommand', 'execute'])
->getMock();
// Mock the getCommand method
$s3ClientMock->expects($this->once())
->method('getCommand')
->with('listBuckets')
->willReturn(new Command('listBuckets'));
// Mock the execute method to throw an S3Exception
$s3ClientMock->expects($this->once())
->method('execute')
->willThrowException(new S3Exception('Error', new Command('ListBuckets')));
// Mock the Wasabi class to return the mocked S3Client
$wasabi = $this->getMockBuilder(Wasabi::class)
->setConstructorArgs([$storageProvider])
->onlyMethods(['getClient'])
->getMock();
$wasabi->expects($this->once())
->method('getClient')
->willReturn($s3ClientMock);
$this->assertFalse($wasabi->connect());
}
}