feature: create multitasking

This commit is contained in:
Daniil
2026-02-04 02:19:50 +03:00
parent 67e0f22b4f
commit a25bf623ea
24 changed files with 5227 additions and 21 deletions
+595
View File
@@ -0,0 +1,595 @@
"""
Unit tests for S3 storage backend.
"""
from __future__ import annotations
import io
from unittest.mock import MagicMock, patch
import pytest
from botocore.exceptions import ClientError
from cpv3.infrastructure.storage.s3 import S3Config, S3StorageBackend
@pytest.fixture
def s3_config() -> S3Config:
"""Create a test S3 configuration."""
return S3Config(
access_key="test-access-key",
secret_key="test-secret-key",
bucket_name="test-bucket",
endpoint_url_internal="http://localhost:9000",
endpoint_url_public="http://localhost:9000",
presign_expires_seconds=3600,
)
@pytest.fixture
def mock_boto3_session():
"""Mock boto3 session and clients."""
with patch("cpv3.infrastructure.storage.s3.boto3.session.Session") as mock_session:
mock_client = MagicMock()
mock_presign_client = MagicMock()
# Track which client is being created
clients = [mock_client, mock_presign_client]
client_index = [0]
def create_client(*args, **kwargs):
idx = client_index[0]
client_index[0] += 1
return clients[idx % 2]
mock_session.return_value.client.side_effect = create_client
yield {
"session": mock_session,
"client": mock_client,
"presign_client": mock_presign_client,
}
@pytest.fixture
def s3_backend(s3_config: S3Config, mock_boto3_session) -> S3StorageBackend:
"""Create an S3StorageBackend with mocked boto3."""
return S3StorageBackend(s3_config)
class TestS3Config:
"""Tests for S3Config dataclass."""
def test_config_creation(self):
"""Test creating S3 config with all parameters."""
config = S3Config(
access_key="key",
secret_key="secret",
bucket_name="bucket",
endpoint_url_internal="http://internal:9000",
endpoint_url_public="http://public:9000",
presign_expires_seconds=7200,
)
assert config.access_key == "key"
assert config.secret_key == "secret"
assert config.bucket_name == "bucket"
assert config.endpoint_url_internal == "http://internal:9000"
assert config.endpoint_url_public == "http://public:9000"
assert config.presign_expires_seconds == 7200
def test_config_default_presign_expires(self):
"""Test default presign expiration time."""
config = S3Config(
access_key="key",
secret_key="secret",
bucket_name="bucket",
endpoint_url_internal=None,
endpoint_url_public=None,
)
assert config.presign_expires_seconds == 3600
def test_config_is_frozen(self):
"""Test that config is immutable."""
config = S3Config(
access_key="key",
secret_key="secret",
bucket_name="bucket",
endpoint_url_internal=None,
endpoint_url_public=None,
)
with pytest.raises(AttributeError):
config.access_key = "new_key" # type: ignore
class TestS3StorageBackendInit:
"""Tests for S3StorageBackend initialization."""
def test_init_creates_two_clients(self, s3_config: S3Config, mock_boto3_session):
"""Test that initialization creates both internal and presign clients."""
S3StorageBackend(s3_config)
# Should create two clients
assert mock_boto3_session["session"].return_value.client.call_count == 2
def test_init_uses_correct_endpoints(self, mock_boto3_session):
"""Test that clients use correct endpoints."""
config = S3Config(
access_key="key",
secret_key="secret",
bucket_name="bucket",
endpoint_url_internal="http://internal:9000",
endpoint_url_public="http://public:9000",
)
S3StorageBackend(config)
calls = mock_boto3_session["session"].return_value.client.call_args_list
# First call should use internal endpoint
assert calls[0][1]["endpoint_url"] == "http://internal:9000"
# Second call should use public endpoint
assert calls[1][1]["endpoint_url"] == "http://public:9000"
def test_init_uses_internal_for_presign_when_no_public(self, mock_boto3_session):
"""Test that presign client uses internal endpoint when public is not set."""
config = S3Config(
access_key="key",
secret_key="secret",
bucket_name="bucket",
endpoint_url_internal="http://internal:9000",
endpoint_url_public=None,
)
S3StorageBackend(config)
calls = mock_boto3_session["session"].return_value.client.call_args_list
# Both should use internal endpoint
assert calls[0][1]["endpoint_url"] == "http://internal:9000"
assert calls[1][1]["endpoint_url"] == "http://internal:9000"
class TestEnsureBucket:
"""Tests for ensure_bucket method."""
def test_ensure_bucket_creates_bucket_if_not_exists(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that bucket is created if it doesn't exist."""
mock_client = mock_boto3_session["client"]
# Simulate bucket not found
error_response = {"Error": {"Code": "404"}}
mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket")
s3_backend.ensure_bucket()
mock_client.create_bucket.assert_called_once_with(Bucket="test-bucket")
def test_ensure_bucket_creates_bucket_no_such_bucket_error(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that bucket is created on NoSuchBucket error."""
mock_client = mock_boto3_session["client"]
error_response = {"Error": {"Code": "NoSuchBucket"}}
mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket")
s3_backend.ensure_bucket()
mock_client.create_bucket.assert_called_once_with(Bucket="test-bucket")
def test_ensure_bucket_does_not_create_if_exists(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that bucket is not created if it already exists."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
s3_backend.ensure_bucket()
mock_client.create_bucket.assert_not_called()
def test_ensure_bucket_caches_result(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that bucket check is cached after first call."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
s3_backend.ensure_bucket()
s3_backend.ensure_bucket()
s3_backend.ensure_bucket()
# Should only check once
assert mock_client.head_bucket.call_count == 1
def test_ensure_bucket_raises_on_other_errors(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that other errors are raised."""
mock_client = mock_boto3_session["client"]
error_response = {"Error": {"Code": "AccessDenied"}}
mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket")
with pytest.raises(ClientError):
s3_backend.ensure_bucket()
class TestUploadFileobj:
"""Tests for upload_fileobj method."""
def test_upload_fileobj_with_content_type(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test uploading a file with content type."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
fileobj = io.BytesIO(b"test content")
s3_backend.upload_fileobj("test/key.txt", fileobj, content_type="text/plain")
mock_client.upload_fileobj.assert_called_once()
call_kwargs = mock_client.upload_fileobj.call_args[1]
assert call_kwargs["Bucket"] == "test-bucket"
assert call_kwargs["Key"] == "test/key.txt"
assert call_kwargs["ExtraArgs"] == {"ContentType": "text/plain"}
def test_upload_fileobj_without_content_type(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test uploading a file without content type."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
fileobj = io.BytesIO(b"test content")
s3_backend.upload_fileobj("test/key.txt", fileobj, content_type=None)
call_kwargs = mock_client.upload_fileobj.call_args[1]
assert call_kwargs["ExtraArgs"] is None
def test_upload_fileobj_ensures_bucket(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that upload_fileobj calls ensure_bucket."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
fileobj = io.BytesIO(b"test content")
s3_backend.upload_fileobj("test/key.txt", fileobj, content_type=None)
mock_client.head_bucket.assert_called_once()
class TestDownloadFileobj:
"""Tests for download_fileobj method."""
def test_download_fileobj(self, s3_backend: S3StorageBackend, mock_boto3_session):
"""Test downloading a file."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
fileobj = io.BytesIO()
s3_backend.download_fileobj("test/key.txt", fileobj)
mock_client.download_fileobj.assert_called_once_with(
"test-bucket", "test/key.txt", fileobj
)
def test_download_fileobj_ensures_bucket(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that download_fileobj calls ensure_bucket."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
fileobj = io.BytesIO()
s3_backend.download_fileobj("test/key.txt", fileobj)
mock_client.head_bucket.assert_called_once()
class TestExists:
"""Tests for exists method."""
def test_exists_returns_true_when_object_exists(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that exists returns True when object exists."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
mock_client.head_object.return_value = {}
result = s3_backend.exists("test/key.txt")
assert result is True
mock_client.head_object.assert_called_once_with(
Bucket="test-bucket", Key="test/key.txt"
)
def test_exists_returns_false_when_object_not_found_404(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that exists returns False on 404 error."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
error_response = {"Error": {"Code": "404"}}
mock_client.head_object.side_effect = ClientError(error_response, "HeadObject")
result = s3_backend.exists("test/key.txt")
assert result is False
def test_exists_returns_false_when_no_such_key(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that exists returns False on NoSuchKey error."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
error_response = {"Error": {"Code": "NoSuchKey"}}
mock_client.head_object.side_effect = ClientError(error_response, "HeadObject")
result = s3_backend.exists("test/key.txt")
assert result is False
def test_exists_raises_on_other_errors(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that other errors are raised."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
error_response = {"Error": {"Code": "AccessDenied"}}
mock_client.head_object.side_effect = ClientError(error_response, "HeadObject")
with pytest.raises(ClientError):
s3_backend.exists("test/key.txt")
class TestSize:
"""Tests for size method."""
def test_size_returns_content_length(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that size returns ContentLength from head_object."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
mock_client.head_object.return_value = {"ContentLength": 12345}
result = s3_backend.size("test/key.txt")
assert result == 12345
mock_client.head_object.assert_called_once_with(
Bucket="test-bucket", Key="test/key.txt"
)
def test_size_returns_zero_when_no_content_length(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that size returns 0 when ContentLength is missing."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
mock_client.head_object.return_value = {}
result = s3_backend.size("test/key.txt")
assert result == 0
class TestDelete:
"""Tests for delete method."""
def test_delete_calls_delete_object(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that delete calls delete_object."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
s3_backend.delete("test/key.txt")
mock_client.delete_object.assert_called_once_with(
Bucket="test-bucket", Key="test/key.txt"
)
def test_delete_ensures_bucket(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that delete calls ensure_bucket."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
s3_backend.delete("test/key.txt")
mock_client.head_bucket.assert_called_once()
class TestRead:
"""Tests for read method."""
def test_read_returns_body_content(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that read returns the object body content."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
mock_body = MagicMock()
mock_body.read.return_value = b"file content"
mock_client.get_object.return_value = {"Body": mock_body}
result = s3_backend.read("test/key.txt")
assert result == b"file content"
mock_client.get_object.assert_called_once_with(
Bucket="test-bucket", Key="test/key.txt"
)
def test_read_ensures_bucket(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that read calls ensure_bucket."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
mock_body = MagicMock()
mock_body.read.return_value = b"content"
mock_client.get_object.return_value = {"Body": mock_body}
s3_backend.read("test/key.txt")
mock_client.head_bucket.assert_called_once()
class TestGenerateUrl:
"""Tests for generate_url method."""
def test_generate_url_returns_presigned_url(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that generate_url returns a presigned URL."""
mock_client = mock_boto3_session["client"]
mock_presign_client = mock_boto3_session["presign_client"]
mock_client.head_bucket.return_value = {}
mock_presign_client.generate_presigned_url.return_value = (
"https://example.com/presigned-url"
)
result = s3_backend.generate_url("test/key.txt")
assert result == "https://example.com/presigned-url"
def test_generate_url_uses_correct_parameters(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test that generate_url uses correct parameters."""
mock_client = mock_boto3_session["client"]
mock_presign_client = mock_boto3_session["presign_client"]
mock_client.head_bucket.return_value = {}
s3_backend.generate_url("test/key.txt")
mock_presign_client.generate_presigned_url.assert_called_once_with(
ClientMethod="get_object",
Params={"Bucket": "test-bucket", "Key": "test/key.txt"},
ExpiresIn=3600,
)
def test_generate_url_uses_custom_expiration(self, mock_boto3_session):
"""Test that generate_url uses custom expiration time."""
config = S3Config(
access_key="key",
secret_key="secret",
bucket_name="bucket",
endpoint_url_internal="http://localhost:9000",
endpoint_url_public=None,
presign_expires_seconds=7200,
)
backend = S3StorageBackend(config)
mock_client = mock_boto3_session["client"]
mock_presign_client = mock_boto3_session["presign_client"]
mock_client.head_bucket.return_value = {}
backend.generate_url("test/key.txt")
call_kwargs = mock_presign_client.generate_presigned_url.call_args[1]
assert call_kwargs["ExpiresIn"] == 7200
class TestIntegrationScenarios:
"""Integration-style tests for common usage scenarios."""
def test_upload_then_check_exists(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test uploading a file and checking it exists."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
mock_client.head_object.return_value = {}
# Upload
fileobj = io.BytesIO(b"test content")
s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain")
# Check exists
result = s3_backend.exists("test/file.txt")
assert result is True
def test_upload_then_read_back(
self, s3_backend: S3StorageBackend, mock_boto3_session
):
"""Test uploading a file and reading it back."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
content = b"test file content"
mock_body = MagicMock()
mock_body.read.return_value = content
mock_client.get_object.return_value = {"Body": mock_body}
# Upload
fileobj = io.BytesIO(content)
s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain")
# Read back
result = s3_backend.read("test/file.txt")
assert result == content
def test_upload_then_delete(self, s3_backend: S3StorageBackend, mock_boto3_session):
"""Test uploading a file and then deleting it."""
mock_client = mock_boto3_session["client"]
mock_client.head_bucket.return_value = {}
# Upload
fileobj = io.BytesIO(b"test content")
s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain")
# Delete
s3_backend.delete("test/file.txt")
mock_client.delete_object.assert_called_once_with(
Bucket="test-bucket", Key="test/file.txt"
)
def test_full_lifecycle(self, s3_backend: S3StorageBackend, mock_boto3_session):
"""Test full file lifecycle: upload, check, get size, get url, delete."""
mock_client = mock_boto3_session["client"]
mock_presign_client = mock_boto3_session["presign_client"]
mock_client.head_bucket.return_value = {}
mock_client.head_object.return_value = {"ContentLength": 100}
mock_presign_client.generate_presigned_url.return_value = "https://url"
content = b"test file content"
# Upload
fileobj = io.BytesIO(content)
s3_backend.upload_fileobj("media/file.mp4", fileobj, content_type="video/mp4")
# Check exists
assert s3_backend.exists("media/file.mp4") is True
# Get size
assert s3_backend.size("media/file.mp4") == 100
# Generate URL
url = s3_backend.generate_url("media/file.mp4")
assert url == "https://url"
# Delete
s3_backend.delete("media/file.mp4")
# Verify all operations were called
assert mock_client.upload_fileobj.called
assert mock_client.delete_object.called