Daniil
596 lines
20 KiB
Python
596 lines
20 KiB
Python
"""
|
|
Unit tests for S3 storage backend.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
from botocore.exceptions import ClientError
|
|
|
|
from cpv3.infrastructure.storage.s3 import S3Config, S3StorageBackend
|
|
|
|
|
|
@pytest.fixture
|
|
def s3_config() -> S3Config:
|
|
"""Create a test S3 configuration."""
|
|
return S3Config(
|
|
access_key="test-access-key",
|
|
secret_key="test-secret-key",
|
|
bucket_name="test-bucket",
|
|
endpoint_url_internal="http://localhost:9000",
|
|
endpoint_url_public="http://localhost:9000",
|
|
presign_expires_seconds=3600,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_boto3_session():
|
|
"""Mock boto3 session and clients."""
|
|
with patch("cpv3.infrastructure.storage.s3.boto3.session.Session") as mock_session:
|
|
mock_client = MagicMock()
|
|
mock_presign_client = MagicMock()
|
|
|
|
# Track which client is being created
|
|
clients = [mock_client, mock_presign_client]
|
|
client_index = [0]
|
|
|
|
def create_client(*args, **kwargs):
|
|
idx = client_index[0]
|
|
client_index[0] += 1
|
|
return clients[idx % 2]
|
|
|
|
mock_session.return_value.client.side_effect = create_client
|
|
|
|
yield {
|
|
"session": mock_session,
|
|
"client": mock_client,
|
|
"presign_client": mock_presign_client,
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def s3_backend(s3_config: S3Config, mock_boto3_session) -> S3StorageBackend:
|
|
"""Create an S3StorageBackend with mocked boto3."""
|
|
return S3StorageBackend(s3_config)
|
|
|
|
|
|
class TestS3Config:
|
|
"""Tests for S3Config dataclass."""
|
|
|
|
def test_config_creation(self):
|
|
"""Test creating S3 config with all parameters."""
|
|
config = S3Config(
|
|
access_key="key",
|
|
secret_key="secret",
|
|
bucket_name="bucket",
|
|
endpoint_url_internal="http://internal:9000",
|
|
endpoint_url_public="http://public:9000",
|
|
presign_expires_seconds=7200,
|
|
)
|
|
|
|
assert config.access_key == "key"
|
|
assert config.secret_key == "secret"
|
|
assert config.bucket_name == "bucket"
|
|
assert config.endpoint_url_internal == "http://internal:9000"
|
|
assert config.endpoint_url_public == "http://public:9000"
|
|
assert config.presign_expires_seconds == 7200
|
|
|
|
def test_config_default_presign_expires(self):
|
|
"""Test default presign expiration time."""
|
|
config = S3Config(
|
|
access_key="key",
|
|
secret_key="secret",
|
|
bucket_name="bucket",
|
|
endpoint_url_internal=None,
|
|
endpoint_url_public=None,
|
|
)
|
|
|
|
assert config.presign_expires_seconds == 3600
|
|
|
|
def test_config_is_frozen(self):
|
|
"""Test that config is immutable."""
|
|
config = S3Config(
|
|
access_key="key",
|
|
secret_key="secret",
|
|
bucket_name="bucket",
|
|
endpoint_url_internal=None,
|
|
endpoint_url_public=None,
|
|
)
|
|
|
|
with pytest.raises(AttributeError):
|
|
config.access_key = "new_key" # type: ignore
|
|
|
|
|
|
class TestS3StorageBackendInit:
|
|
"""Tests for S3StorageBackend initialization."""
|
|
|
|
def test_init_creates_two_clients(self, s3_config: S3Config, mock_boto3_session):
|
|
"""Test that initialization creates both internal and presign clients."""
|
|
S3StorageBackend(s3_config)
|
|
|
|
# Should create two clients
|
|
assert mock_boto3_session["session"].return_value.client.call_count == 2
|
|
|
|
def test_init_uses_correct_endpoints(self, mock_boto3_session):
|
|
"""Test that clients use correct endpoints."""
|
|
config = S3Config(
|
|
access_key="key",
|
|
secret_key="secret",
|
|
bucket_name="bucket",
|
|
endpoint_url_internal="http://internal:9000",
|
|
endpoint_url_public="http://public:9000",
|
|
)
|
|
|
|
S3StorageBackend(config)
|
|
|
|
calls = mock_boto3_session["session"].return_value.client.call_args_list
|
|
|
|
# First call should use internal endpoint
|
|
assert calls[0][1]["endpoint_url"] == "http://internal:9000"
|
|
# Second call should use public endpoint
|
|
assert calls[1][1]["endpoint_url"] == "http://public:9000"
|
|
|
|
def test_init_uses_internal_for_presign_when_no_public(self, mock_boto3_session):
|
|
"""Test that presign client uses internal endpoint when public is not set."""
|
|
config = S3Config(
|
|
access_key="key",
|
|
secret_key="secret",
|
|
bucket_name="bucket",
|
|
endpoint_url_internal="http://internal:9000",
|
|
endpoint_url_public=None,
|
|
)
|
|
|
|
S3StorageBackend(config)
|
|
|
|
calls = mock_boto3_session["session"].return_value.client.call_args_list
|
|
|
|
# Both should use internal endpoint
|
|
assert calls[0][1]["endpoint_url"] == "http://internal:9000"
|
|
assert calls[1][1]["endpoint_url"] == "http://internal:9000"
|
|
|
|
|
|
class TestEnsureBucket:
|
|
"""Tests for ensure_bucket method."""
|
|
|
|
def test_ensure_bucket_creates_bucket_if_not_exists(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that bucket is created if it doesn't exist."""
|
|
mock_client = mock_boto3_session["client"]
|
|
|
|
# Simulate bucket not found
|
|
error_response = {"Error": {"Code": "404"}}
|
|
mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket")
|
|
|
|
s3_backend.ensure_bucket()
|
|
|
|
mock_client.create_bucket.assert_called_once_with(Bucket="test-bucket")
|
|
|
|
def test_ensure_bucket_creates_bucket_no_such_bucket_error(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that bucket is created on NoSuchBucket error."""
|
|
mock_client = mock_boto3_session["client"]
|
|
|
|
error_response = {"Error": {"Code": "NoSuchBucket"}}
|
|
mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket")
|
|
|
|
s3_backend.ensure_bucket()
|
|
|
|
mock_client.create_bucket.assert_called_once_with(Bucket="test-bucket")
|
|
|
|
def test_ensure_bucket_does_not_create_if_exists(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that bucket is not created if it already exists."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
s3_backend.ensure_bucket()
|
|
|
|
mock_client.create_bucket.assert_not_called()
|
|
|
|
def test_ensure_bucket_caches_result(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that bucket check is cached after first call."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
s3_backend.ensure_bucket()
|
|
s3_backend.ensure_bucket()
|
|
s3_backend.ensure_bucket()
|
|
|
|
# Should only check once
|
|
assert mock_client.head_bucket.call_count == 1
|
|
|
|
def test_ensure_bucket_raises_on_other_errors(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that other errors are raised."""
|
|
mock_client = mock_boto3_session["client"]
|
|
|
|
error_response = {"Error": {"Code": "AccessDenied"}}
|
|
mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket")
|
|
|
|
with pytest.raises(ClientError):
|
|
s3_backend.ensure_bucket()
|
|
|
|
|
|
class TestUploadFileobj:
|
|
"""Tests for upload_fileobj method."""
|
|
|
|
def test_upload_fileobj_with_content_type(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test uploading a file with content type."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
fileobj = io.BytesIO(b"test content")
|
|
s3_backend.upload_fileobj("test/key.txt", fileobj, content_type="text/plain")
|
|
|
|
mock_client.upload_fileobj.assert_called_once()
|
|
call_kwargs = mock_client.upload_fileobj.call_args[1]
|
|
|
|
assert call_kwargs["Bucket"] == "test-bucket"
|
|
assert call_kwargs["Key"] == "test/key.txt"
|
|
assert call_kwargs["ExtraArgs"] == {"ContentType": "text/plain"}
|
|
|
|
def test_upload_fileobj_without_content_type(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test uploading a file without content type."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
fileobj = io.BytesIO(b"test content")
|
|
s3_backend.upload_fileobj("test/key.txt", fileobj, content_type=None)
|
|
|
|
call_kwargs = mock_client.upload_fileobj.call_args[1]
|
|
assert call_kwargs["ExtraArgs"] is None
|
|
|
|
def test_upload_fileobj_ensures_bucket(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that upload_fileobj calls ensure_bucket."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
fileobj = io.BytesIO(b"test content")
|
|
s3_backend.upload_fileobj("test/key.txt", fileobj, content_type=None)
|
|
|
|
mock_client.head_bucket.assert_called_once()
|
|
|
|
|
|
class TestDownloadFileobj:
|
|
"""Tests for download_fileobj method."""
|
|
|
|
def test_download_fileobj(self, s3_backend: S3StorageBackend, mock_boto3_session):
|
|
"""Test downloading a file."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
fileobj = io.BytesIO()
|
|
s3_backend.download_fileobj("test/key.txt", fileobj)
|
|
|
|
mock_client.download_fileobj.assert_called_once_with(
|
|
"test-bucket", "test/key.txt", fileobj
|
|
)
|
|
|
|
def test_download_fileobj_ensures_bucket(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that download_fileobj calls ensure_bucket."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
fileobj = io.BytesIO()
|
|
s3_backend.download_fileobj("test/key.txt", fileobj)
|
|
|
|
mock_client.head_bucket.assert_called_once()
|
|
|
|
|
|
class TestExists:
|
|
"""Tests for exists method."""
|
|
|
|
def test_exists_returns_true_when_object_exists(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that exists returns True when object exists."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
mock_client.head_object.return_value = {}
|
|
|
|
result = s3_backend.exists("test/key.txt")
|
|
|
|
assert result is True
|
|
mock_client.head_object.assert_called_once_with(
|
|
Bucket="test-bucket", Key="test/key.txt"
|
|
)
|
|
|
|
def test_exists_returns_false_when_object_not_found_404(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that exists returns False on 404 error."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
error_response = {"Error": {"Code": "404"}}
|
|
mock_client.head_object.side_effect = ClientError(error_response, "HeadObject")
|
|
|
|
result = s3_backend.exists("test/key.txt")
|
|
|
|
assert result is False
|
|
|
|
def test_exists_returns_false_when_no_such_key(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that exists returns False on NoSuchKey error."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
error_response = {"Error": {"Code": "NoSuchKey"}}
|
|
mock_client.head_object.side_effect = ClientError(error_response, "HeadObject")
|
|
|
|
result = s3_backend.exists("test/key.txt")
|
|
|
|
assert result is False
|
|
|
|
def test_exists_raises_on_other_errors(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that other errors are raised."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
error_response = {"Error": {"Code": "AccessDenied"}}
|
|
mock_client.head_object.side_effect = ClientError(error_response, "HeadObject")
|
|
|
|
with pytest.raises(ClientError):
|
|
s3_backend.exists("test/key.txt")
|
|
|
|
|
|
class TestSize:
|
|
"""Tests for size method."""
|
|
|
|
def test_size_returns_content_length(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that size returns ContentLength from head_object."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
mock_client.head_object.return_value = {"ContentLength": 12345}
|
|
|
|
result = s3_backend.size("test/key.txt")
|
|
|
|
assert result == 12345
|
|
mock_client.head_object.assert_called_once_with(
|
|
Bucket="test-bucket", Key="test/key.txt"
|
|
)
|
|
|
|
def test_size_returns_zero_when_no_content_length(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that size returns 0 when ContentLength is missing."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
mock_client.head_object.return_value = {}
|
|
|
|
result = s3_backend.size("test/key.txt")
|
|
|
|
assert result == 0
|
|
|
|
|
|
class TestDelete:
|
|
"""Tests for delete method."""
|
|
|
|
def test_delete_calls_delete_object(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that delete calls delete_object."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
s3_backend.delete("test/key.txt")
|
|
|
|
mock_client.delete_object.assert_called_once_with(
|
|
Bucket="test-bucket", Key="test/key.txt"
|
|
)
|
|
|
|
def test_delete_ensures_bucket(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that delete calls ensure_bucket."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
s3_backend.delete("test/key.txt")
|
|
|
|
mock_client.head_bucket.assert_called_once()
|
|
|
|
|
|
class TestRead:
|
|
"""Tests for read method."""
|
|
|
|
def test_read_returns_body_content(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that read returns the object body content."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
mock_body = MagicMock()
|
|
mock_body.read.return_value = b"file content"
|
|
mock_client.get_object.return_value = {"Body": mock_body}
|
|
|
|
result = s3_backend.read("test/key.txt")
|
|
|
|
assert result == b"file content"
|
|
mock_client.get_object.assert_called_once_with(
|
|
Bucket="test-bucket", Key="test/key.txt"
|
|
)
|
|
|
|
def test_read_ensures_bucket(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that read calls ensure_bucket."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
mock_body = MagicMock()
|
|
mock_body.read.return_value = b"content"
|
|
mock_client.get_object.return_value = {"Body": mock_body}
|
|
|
|
s3_backend.read("test/key.txt")
|
|
|
|
mock_client.head_bucket.assert_called_once()
|
|
|
|
|
|
class TestGenerateUrl:
|
|
"""Tests for generate_url method."""
|
|
|
|
def test_generate_url_returns_presigned_url(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that generate_url returns a presigned URL."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_presign_client = mock_boto3_session["presign_client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
mock_presign_client.generate_presigned_url.return_value = (
|
|
"https://example.com/presigned-url"
|
|
)
|
|
|
|
result = s3_backend.generate_url("test/key.txt")
|
|
|
|
assert result == "https://example.com/presigned-url"
|
|
|
|
def test_generate_url_uses_correct_parameters(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test that generate_url uses correct parameters."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_presign_client = mock_boto3_session["presign_client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
s3_backend.generate_url("test/key.txt")
|
|
|
|
mock_presign_client.generate_presigned_url.assert_called_once_with(
|
|
ClientMethod="get_object",
|
|
Params={"Bucket": "test-bucket", "Key": "test/key.txt"},
|
|
ExpiresIn=3600,
|
|
)
|
|
|
|
def test_generate_url_uses_custom_expiration(self, mock_boto3_session):
|
|
"""Test that generate_url uses custom expiration time."""
|
|
config = S3Config(
|
|
access_key="key",
|
|
secret_key="secret",
|
|
bucket_name="bucket",
|
|
endpoint_url_internal="http://localhost:9000",
|
|
endpoint_url_public=None,
|
|
presign_expires_seconds=7200,
|
|
)
|
|
|
|
backend = S3StorageBackend(config)
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_presign_client = mock_boto3_session["presign_client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
backend.generate_url("test/key.txt")
|
|
|
|
call_kwargs = mock_presign_client.generate_presigned_url.call_args[1]
|
|
assert call_kwargs["ExpiresIn"] == 7200
|
|
|
|
|
|
class TestIntegrationScenarios:
|
|
"""Integration-style tests for common usage scenarios."""
|
|
|
|
def test_upload_then_check_exists(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test uploading a file and checking it exists."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
mock_client.head_object.return_value = {}
|
|
|
|
# Upload
|
|
fileobj = io.BytesIO(b"test content")
|
|
s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain")
|
|
|
|
# Check exists
|
|
result = s3_backend.exists("test/file.txt")
|
|
|
|
assert result is True
|
|
|
|
def test_upload_then_read_back(
|
|
self, s3_backend: S3StorageBackend, mock_boto3_session
|
|
):
|
|
"""Test uploading a file and reading it back."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
content = b"test file content"
|
|
mock_body = MagicMock()
|
|
mock_body.read.return_value = content
|
|
mock_client.get_object.return_value = {"Body": mock_body}
|
|
|
|
# Upload
|
|
fileobj = io.BytesIO(content)
|
|
s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain")
|
|
|
|
# Read back
|
|
result = s3_backend.read("test/file.txt")
|
|
|
|
assert result == content
|
|
|
|
def test_upload_then_delete(self, s3_backend: S3StorageBackend, mock_boto3_session):
|
|
"""Test uploading a file and then deleting it."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
|
|
# Upload
|
|
fileobj = io.BytesIO(b"test content")
|
|
s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain")
|
|
|
|
# Delete
|
|
s3_backend.delete("test/file.txt")
|
|
|
|
mock_client.delete_object.assert_called_once_with(
|
|
Bucket="test-bucket", Key="test/file.txt"
|
|
)
|
|
|
|
def test_full_lifecycle(self, s3_backend: S3StorageBackend, mock_boto3_session):
|
|
"""Test full file lifecycle: upload, check, get size, get url, delete."""
|
|
mock_client = mock_boto3_session["client"]
|
|
mock_presign_client = mock_boto3_session["presign_client"]
|
|
mock_client.head_bucket.return_value = {}
|
|
mock_client.head_object.return_value = {"ContentLength": 100}
|
|
mock_presign_client.generate_presigned_url.return_value = "https://url"
|
|
|
|
content = b"test file content"
|
|
|
|
# Upload
|
|
fileobj = io.BytesIO(content)
|
|
s3_backend.upload_fileobj("media/file.mp4", fileobj, content_type="video/mp4")
|
|
|
|
# Check exists
|
|
assert s3_backend.exists("media/file.mp4") is True
|
|
|
|
# Get size
|
|
assert s3_backend.size("media/file.mp4") == 100
|
|
|
|
# Generate URL
|
|
url = s3_backend.generate_url("media/file.mp4")
|
|
assert url == "https://url"
|
|
|
|
# Delete
|
|
s3_backend.delete("media/file.mp4")
|
|
|
|
# Verify all operations were called
|
|
assert mock_client.upload_fileobj.called
|
|
assert mock_client.delete_object.called
|