From 9a8945b0dc77d9bb43a28fa494866c371451ee86 Mon Sep 17 00:00:00 2001 From: Constantin Graf Date: Mon, 9 Sep 2024 22:30:24 +0200 Subject: [PATCH] Add local setup for S3 --- .env.example | 18 +++--- config/filesystems.php | 1 + docker-compose.yml | 62 ++++++++++++++++++- docker/local/minio/create_bucket.sh | 10 +++ tests/TestCase.php | 11 ++++ .../Endpoint/Api/V1/ExportEndpointTest.php | 10 +-- tests/Unit/Service/DeletionServiceTest.php | 9 +-- .../Unit/Service/Export/ExportServiceTest.php | 3 +- .../TogglTimeEntriesImporterTest.php | 9 +++ 9 files changed, 115 insertions(+), 18 deletions(-) create mode 100755 docker/local/minio/create_bucket.sh diff --git a/.env.example b/.env.example index 15a3e025..265804e9 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,6 @@ DB_TEST_PASSWORD=root BROADCAST_DRIVER=log CACHE_DRIVER=file -FILESYSTEM_DISK=local QUEUE_CONNECTION=sync SESSION_DRIVER=database SESSION_LIFETIME=120 @@ -47,12 +46,6 @@ MAIL_ENCRYPTION=null MAIL_FROM_ADDRESS="no-reply@solidtime.test" MAIL_FROM_NAME="${APP_NAME}" -S3_ACCESS_KEY_ID= -S3_SECRET_ACCESS_KEY= -S3_REGION=us-east-1 -S3_BUCKET= -S3_USE_PATH_STYLE_ENDPOINT=false - PUSHER_APP_ID= PUSHER_APP_KEY= PUSHER_APP_SECRET= @@ -61,6 +54,17 @@ PUSHER_PORT=443 PUSHER_SCHEME=https PUSHER_APP_CLUSTER=mt1 +# Storage +FILESYSTEM_DISK=s3 +PUBLIC_FILESYSTEM_DISK=s3 +S3_ACCESS_KEY_ID=sail +S3_SECRET_ACCESS_KEY=password +S3_REGION=us-east-1 +S3_BUCKET=local +S3_URL=http://storage.solidtime.test/local +S3_ENDPOINT=http://storage.solidtime.test +S3_USE_PATH_STYLE_ENDPOINT=true + VITE_HOST_NAME=vite.solidtime.test VITE_APP_NAME="${APP_NAME}" VITE_PUSHER_APP_KEY="${PUSHER_APP_KEY}" diff --git a/config/filesystems.php b/config/filesystems.php index 05648826..55d8df73 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -57,6 +57,7 @@ return [ 'region' => env('S3_REGION'), 'bucket' => env('S3_BUCKET'), 'url' => env('S3_URL'), + 'temporary_url' => env('S3_URL'), 'endpoint' => env('S3_ENDPOINT'), 'use_path_style_endpoint' => env('S3_USE_PATH_STYLE_ENDPOINT', false), 'throw' => true, diff --git a/docker-compose.yml b/docker-compose.yml index 389d7264..ca272ec4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,8 @@ services: - "traefik.http.routers.solidtime-dev-vite.service=solidtime-dev-vite" - "traefik.http.routers.solidtime-dev-vite.entrypoints=web" extra_hosts: - - 'host.docker.internal:host-gateway' + - "host.docker.internal:host-gateway" + - "storage.${NGINX_HOST_NAME}:${REVERSE_PROXY_IP:-10.100.100.10}" environment: XDG_CONFIG_HOME: /var/www/html/config XDG_DATA_HOME: /var/www/html/data @@ -128,6 +129,63 @@ services: - reverse-proxy volumes: - '.:/src' + minio: + image: 'minio/minio:latest' + environment: + MINIO_BROWSER_REDIRECT_URL: 'https://storage-management.${NGINX_HOST_NAME}' + MINIO_ROOT_USER: 'sail' + MINIO_ROOT_PASSWORD: 'password' + volumes: + - 'sail-minio:/data/minio' + networks: + - reverse-proxy + - sail + command: minio server /data/minio --console-address ":8900" + healthcheck: + test: [ "CMD", "mc", "ready", "local" ] + interval: 5s + timeout: 5s + retries: 5 + labels: + - "traefik.enable=true" + - "traefik.docker.network=${NETWORK_NAME}" + # Storage Frontend + - "traefik.http.services.solidtime-dev-storage-frontend.loadbalancer.server.port=9000" + # http + - "traefik.http.routers.solidtime-dev-storage-frontend.rule=Host(`storage.${NGINX_HOST_NAME}`)" + - "traefik.http.routers.solidtime-dev-storage-frontend.service=solidtime-dev-storage-frontend" + - "traefik.http.routers.solidtime-dev-storage-frontend.entrypoints=web" + # https + - "traefik.http.routers.solidtime-dev-storage-frontend-https.rule=Host(`storage.${NGINX_HOST_NAME}`)" + - "traefik.http.routers.solidtime-dev-storage-frontend-https.service=solidtime-dev-storage-frontend" + - "traefik.http.routers.solidtime-dev-storage-frontend-https.entrypoints=websecure" + - "traefik.http.routers.solidtime-dev-storage-frontend-https.tls=true" + # Storage Management + - "traefik.http.services.solidtime-dev-storage-management.loadbalancer.server.port=8900" + # http + - "traefik.http.routers.solidtime-dev-storage-management.rule=Host(`storage-management.${NGINX_HOST_NAME}`)" + - "traefik.http.routers.solidtime-dev-storage-management.service=solidtime-dev-storage-management" + - "traefik.http.routers.solidtime-dev-storage-management.entrypoints=web" + # https + - "traefik.http.routers.solidtime-dev-storage-management-https.rule=Host(`storage-management.${NGINX_HOST_NAME}`)" + - "traefik.http.routers.solidtime-dev-storage-management-https.service=solidtime-dev-storage-management" + - "traefik.http.routers.solidtime-dev-storage-management-https.entrypoints=websecure" + - "traefik.http.routers.solidtime-dev-storage-management-https.tls=true" + + minio-create-bucket: + image: minio/mc:latest + depends_on: + - minio + environment: + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_BUCKET: ${S3_BUCKET} + S3_ENDPOINT: ${S3_ENDPOINT} + volumes: + - './docker/local/minio:/etc/minio' + networks: + - sail + entrypoint: /etc/minio/create_bucket.sh networks: reverse-proxy: name: "${NETWORK_NAME}" @@ -139,3 +197,5 @@ volumes: driver: local sail-pgsql-test: driver: local + sail-minio: + driver: local diff --git a/docker/local/minio/create_bucket.sh b/docker/local/minio/create_bucket.sh new file mode 100755 index 00000000..944a2520 --- /dev/null +++ b/docker/local/minio/create_bucket.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Source: https://helgesver.re/articles/laravel-sail-create-minio-bucket-automatically + +/usr/bin/mc config host add local ${S3_ENDPOINT} ${S3_ACCESS_KEY_ID} ${S3_SECRET_ACCESS_KEY}; +/usr/bin/mc rm -r --force local/${S3_BUCKET}; +/usr/bin/mc mb --ignore-existing local/${S3_BUCKET}; +/usr/bin/mc anonymous set public local/${S3_BUCKET}; + +exit 0; diff --git a/tests/TestCase.php b/tests/TestCase.php index 553173bb..6679f627 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -11,6 +11,7 @@ use Illuminate\Database\Eloquent\Collection; use Illuminate\Foundation\Testing\TestCase as BaseTestCase; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Mail; +use Illuminate\Support\Facades\Storage; use Mockery\MockInterface; use TiMacDonald\Log\LogFake; @@ -25,6 +26,16 @@ abstract class TestCase extends BaseTestCase LogFake::bind(); } + protected function mockPrivateStorage(): void + { + Storage::fake(config('filesystems.default')); + } + + protected function mockPublicStorage(): void + { + Storage::fake(config('filesystems.public')); + } + protected function tearDown(): void { // Note: It is necessary to clear the permission cache after each test, since the "scoped singletons" are not reset between tests. diff --git a/tests/Unit/Endpoint/Api/V1/ExportEndpointTest.php b/tests/Unit/Endpoint/Api/V1/ExportEndpointTest.php index 71be8660..b20e9cc1 100644 --- a/tests/Unit/Endpoint/Api/V1/ExportEndpointTest.php +++ b/tests/Unit/Endpoint/Api/V1/ExportEndpointTest.php @@ -81,13 +81,13 @@ class ExportEndpointTest extends ApiEndpointTestAbstract Passport::actingAs($user->user); // Act - $response = $this->postJson(route('api.v1.export.export', ['organization' => $user->organization->getKey()])); + $response = $this->postJson(route('api.v1.export.export', [ + 'organization' => $user->organization->getKey(), + ])); // Assert $response->assertStatus(200); - $response->assertExactJson([ - 'success' => true, - 'download_url' => Storage::disk('local')->temporaryUrl($filepath, $now->addMinutes(10)), - ]); + $response->assertJsonPath('success', true); + $this->assertStringContainsString($filepath, $response->json('download_url')); } } diff --git a/tests/Unit/Service/DeletionServiceTest.php b/tests/Unit/Service/DeletionServiceTest.php index cab34f1c..b4b4af55 100644 --- a/tests/Unit/Service/DeletionServiceTest.php +++ b/tests/Unit/Service/DeletionServiceTest.php @@ -261,10 +261,11 @@ class DeletionServiceTest extends TestCaseWithDatabase public function test_delete_user_deletes_all_resources_of_the_user_but_does_not_delete_other_resources(): void { // Arrange + $this->mockPublicStorage(); $user = User::factory()->withProfilePicture()->withPersonalOrganization()->create(); $otherUser = User::factory()->withProfilePicture()->withPersonalOrganization()->create(); - Storage::disk('public')->assertExists($user->profile_photo_path); - Storage::disk('public')->assertExists($otherUser->profile_photo_path); + Storage::disk(config('filesystems.public'))->assertExists($user->profile_photo_path); + Storage::disk(config('filesystems.public'))->assertExists($otherUser->profile_photo_path); // Act $this->deletionService->deleteUser($user); @@ -288,8 +289,8 @@ class DeletionServiceTest extends TestCaseWithDatabase $this->assertDatabaseMissing(Member::class, [ 'user_id' => $user->getKey(), ]); - Storage::disk('public')->assertMissing($user->profile_photo_path); - Storage::disk('public')->assertExists($otherUser->profile_photo_path); + Storage::disk(config('filesystems.public'))->assertMissing($user->profile_photo_path); + Storage::disk(config('filesystems.public'))->assertExists($otherUser->profile_photo_path); Log::assertLoggedTimes(fn (LogEntry $log) => $log->level === 'debug' && $log->message === 'Start deleting user' && $log->context['id'] === $user->getKey(), diff --git a/tests/Unit/Service/Export/ExportServiceTest.php b/tests/Unit/Service/Export/ExportServiceTest.php index f010883d..5137ab2e 100644 --- a/tests/Unit/Service/Export/ExportServiceTest.php +++ b/tests/Unit/Service/Export/ExportServiceTest.php @@ -53,6 +53,7 @@ class ExportServiceTest extends TestCaseWithDatabase public function test_export_creates_zip_with_all_the_data_of_the_organization(): void { // Arrange + $this->mockPrivateStorage(); $organization1 = $this->getFullOrganization(); $organization2 = $this->getFullOrganization(); @@ -61,6 +62,6 @@ class ExportServiceTest extends TestCaseWithDatabase $zip = $exportService->export($organization1); // Assert - Storage::disk('local')->assertExists($zip); + Storage::disk(config('filesystems.default'))->assertExists($zip); } } diff --git a/tests/Unit/Service/Import/Importers/TogglTimeEntriesImporterTest.php b/tests/Unit/Service/Import/Importers/TogglTimeEntriesImporterTest.php index 1622cd04..386087fb 100644 --- a/tests/Unit/Service/Import/Importers/TogglTimeEntriesImporterTest.php +++ b/tests/Unit/Service/Import/Importers/TogglTimeEntriesImporterTest.php @@ -8,6 +8,7 @@ use App\Models\Organization; use App\Service\Import\Importers\DefaultImporter; use App\Service\Import\Importers\ImportException; use App\Service\Import\Importers\TogglTimeEntriesImporter; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Storage; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\UsesClass; @@ -28,10 +29,14 @@ class TogglTimeEntriesImporterTest extends ImporterTestAbstract $data = Storage::disk('testfiles')->get('toggl_time_entries_import_test_1.csv'); // Act + DB::enableQueryLog(); + DB::flushQueryLog(); $importer->importData($data, $timezone); $report = $importer->getReport(); + $queryLog = DB::getQueryLog(); // Assert + $this->assertCount(31, $queryLog); $testScenario = $this->checkTestScenarioAfterImportExcludingTimeEntries(); $this->checkTimeEntries($testScenario); $this->assertSame(2, $report->timeEntriesCreated); @@ -55,10 +60,14 @@ class TogglTimeEntriesImporterTest extends ImporterTestAbstract $importer->init($organization); // Act + DB::enableQueryLog(); + DB::flushQueryLog(); $importer->importData($data, $timezone); $report = $importer->getReport(); + $queryLog = DB::getQueryLog(); // Assert + $this->assertCount(15, $queryLog); $testScenario = $this->checkTestScenarioAfterImportExcludingTimeEntries(); $this->checkTimeEntries($testScenario, true); $this->assertSame(2, $report->timeEntriesCreated);