From 15a4edd25452cc6e6de0401ba86d9a2604c26c10 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 1 Oct 2025 13:46:29 +0200 Subject: [PATCH 1/2] feat: add support for s3 regions, fix type annotations --- .../migrations/0075_s3storagesource_region.py | 17 +++++++++++++++ ami/main/models.py | 2 ++ ami/tests/fixtures/storage.py | 1 + ami/utils/s3.py | 21 ++++++++++++++++--- config/settings/base.py | 1 + 5 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 ami/main/migrations/0075_s3storagesource_region.py diff --git a/ami/main/migrations/0075_s3storagesource_region.py b/ami/main/migrations/0075_s3storagesource_region.py new file mode 100644 index 000000000..23bff2520 --- /dev/null +++ b/ami/main/migrations/0075_s3storagesource_region.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.10 on 2025-09-26 08:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0074_taxon_cover_image_credit_taxon_cover_image_url_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="s3storagesource", + name="region", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 515f5286a..dc12b030c 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -1394,6 +1394,7 @@ class S3StorageSource(BaseModel): name = models.CharField(max_length=255) bucket = models.CharField(max_length=255) + region = models.CharField(max_length=255, null=True, blank=True) prefix = models.CharField(max_length=255, blank=True) access_key = models.TextField() secret_key = models.TextField() @@ -1413,6 +1414,7 @@ class S3StorageSource(BaseModel): def config(self) -> ami.utils.s3.S3Config: return ami.utils.s3.S3Config( bucket_name=self.bucket, + region=self.region, prefix=self.prefix, access_key_id=self.access_key, secret_access_key=self.secret_key, diff --git a/ami/tests/fixtures/storage.py b/ami/tests/fixtures/storage.py index d53c782ff..1e23a25e9 100644 --- a/ami/tests/fixtures/storage.py +++ b/ami/tests/fixtures/storage.py @@ -15,6 +15,7 @@ access_key_id=settings.S3_TEST_KEY, secret_access_key=settings.S3_TEST_SECRET, bucket_name=settings.S3_TEST_BUCKET, + region=settings.S3_TEST_REGION, prefix="test_prefix", public_base_url=f"http://minio:9000/{settings.S3_TEST_BUCKET}/test_prefix", # public_base_url="http://minio:9001", diff --git a/ami/utils/s3.py b/ami/utils/s3.py index ce157b213..4a54c8244 100644 --- a/ami/utils/s3.py +++ b/ami/utils/s3.py @@ -10,7 +10,7 @@ from dataclasses import dataclass import boto3 -import boto3.resources.base +import boto3.session import botocore import botocore.config import botocore.exceptions @@ -37,6 +37,7 @@ class S3Config: secret_access_key: str bucket_name: str prefix: str + region: str | None = None public_base_url: str | None = None sensitive_fields = ["access_key_id", "secret_access_key"] @@ -94,26 +95,36 @@ def get_session(config: S3Config) -> boto3.session.Session: session = boto3.Session( aws_access_key_id=config.access_key_id, aws_secret_access_key=config.secret_access_key, + region_name=config.region, ) return session def get_s3_client(config: S3Config) -> S3Client: session = get_session(config) + + # Always use signature version 4 + boto_config = botocore.config.Config(signature_version="s3v4") + if config.endpoint_url: client = session.client( service_name="s3", endpoint_url=config.endpoint_url, aws_access_key_id=config.access_key_id, aws_secret_access_key=config.secret_access_key, - config=botocore.config.Config(signature_version="s3v4"), + region_name=config.region, + config=boto_config, ) else: client = session.client( service_name="s3", aws_access_key_id=config.access_key_id, aws_secret_access_key=config.secret_access_key, + region_name=config.region, + config=boto_config, ) + + client = typing.cast(S3Client, client) return client @@ -124,6 +135,7 @@ def get_resource(config: S3Config) -> S3ServiceResource: endpoint_url=config.endpoint_url, # api_version="s3v4", ) + s3 = typing.cast(S3ServiceResource, s3) return s3 @@ -584,7 +596,9 @@ def read_image(config: S3Config, key: str) -> PIL.Image.Image: obj = bucket.Object(key) logger.info(f"Fetching image {key} from S3") try: - img = PIL.Image.open(obj.get()["Body"]) + # StreamingBody inherits from io.IOBase, but type checkers don't see that + fp = obj.get()["Body"] + img = PIL.Image.open(fp) # type: ignore[arg-type] except PIL.UnidentifiedImageError: logger.error(f"Could not read image {key}") raise @@ -677,6 +691,7 @@ def test(): bucket_name="test", prefix="", public_base_url="http://minio:9000/test", + region=None, ) projects = list_projects(config) diff --git a/config/settings/base.py b/config/settings/base.py index c9a8a9681..8385c38f3 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -449,6 +449,7 @@ S3_TEST_KEY = env("MINIO_ROOT_USER", default=None) # type: ignore[no-untyped-call] S3_TEST_SECRET = env("MINIO_ROOT_PASSWORD", default=None) # type: ignore[no-untyped-call] S3_TEST_BUCKET = env("MINIO_TEST_BUCKET", default="ami-test") # type: ignore[no-untyped-call] +S3_TEST_REGION = env("MINIO_REGION", default=None) # type: ignore[no-untyped-call] # Default processing service settings From e2e3705710ff55940bc36a6d7c7abadaebb448cb Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Fri, 5 Dec 2025 17:03:10 -0800 Subject: [PATCH 2/2] fix: merge migrations --- ...storagesource_region.py => 0079_s3storagesource_region.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename ami/main/migrations/{0075_s3storagesource_region.py => 0079_s3storagesource_region.py} (70%) diff --git a/ami/main/migrations/0075_s3storagesource_region.py b/ami/main/migrations/0079_s3storagesource_region.py similarity index 70% rename from ami/main/migrations/0075_s3storagesource_region.py rename to ami/main/migrations/0079_s3storagesource_region.py index 23bff2520..c62db56f2 100644 --- a/ami/main/migrations/0075_s3storagesource_region.py +++ b/ami/main/migrations/0079_s3storagesource_region.py @@ -1,11 +1,11 @@ -# Generated by Django 4.2.10 on 2025-09-26 08:57 +# Generated by Django 4.2.10 on 2025-12-05 20:03 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("main", "0074_taxon_cover_image_credit_taxon_cover_image_url_and_more"), + ("main", "0078_classification_applied_to"), ] operations = [