""" Unit tests for S3 storage backend. """ from __future__ import annotations import io from unittest.mock import MagicMock, patch import pytest from botocore.exceptions import ClientError from cpv3.infrastructure.storage.s3 import S3Config, S3StorageBackend @pytest.fixture def s3_config() -> S3Config: """Create a test S3 configuration.""" return S3Config( access_key="test-access-key", secret_key="test-secret-key", bucket_name="test-bucket", endpoint_url_internal="http://localhost:9000", endpoint_url_public="http://localhost:9000", presign_expires_seconds=3600, ) @pytest.fixture def mock_boto3_session(): """Mock boto3 session and clients.""" with patch("cpv3.infrastructure.storage.s3.boto3.session.Session") as mock_session: mock_client = MagicMock() mock_presign_client = MagicMock() # Track which client is being created clients = [mock_client, mock_presign_client] client_index = [0] def create_client(*args, **kwargs): idx = client_index[0] client_index[0] += 1 return clients[idx % 2] mock_session.return_value.client.side_effect = create_client yield { "session": mock_session, "client": mock_client, "presign_client": mock_presign_client, } @pytest.fixture def s3_backend(s3_config: S3Config, mock_boto3_session) -> S3StorageBackend: """Create an S3StorageBackend with mocked boto3.""" return S3StorageBackend(s3_config) class TestS3Config: """Tests for S3Config dataclass.""" def test_config_creation(self): """Test creating S3 config with all parameters.""" config = S3Config( access_key="key", secret_key="secret", bucket_name="bucket", endpoint_url_internal="http://internal:9000", endpoint_url_public="http://public:9000", presign_expires_seconds=7200, ) assert config.access_key == "key" assert config.secret_key == "secret" assert config.bucket_name == "bucket" assert config.endpoint_url_internal == "http://internal:9000" assert config.endpoint_url_public == "http://public:9000" assert config.presign_expires_seconds == 7200 def test_config_default_presign_expires(self): """Test default presign expiration time.""" config = S3Config( access_key="key", secret_key="secret", bucket_name="bucket", endpoint_url_internal=None, endpoint_url_public=None, ) assert config.presign_expires_seconds == 3600 def test_config_is_frozen(self): """Test that config is immutable.""" config = S3Config( access_key="key", secret_key="secret", bucket_name="bucket", endpoint_url_internal=None, endpoint_url_public=None, ) with pytest.raises(AttributeError): config.access_key = "new_key" # type: ignore class TestS3StorageBackendInit: """Tests for S3StorageBackend initialization.""" def test_init_creates_two_clients(self, s3_config: S3Config, mock_boto3_session): """Test that initialization creates both internal and presign clients.""" S3StorageBackend(s3_config) # Should create two clients assert mock_boto3_session["session"].return_value.client.call_count == 2 def test_init_uses_correct_endpoints(self, mock_boto3_session): """Test that clients use correct endpoints.""" config = S3Config( access_key="key", secret_key="secret", bucket_name="bucket", endpoint_url_internal="http://internal:9000", endpoint_url_public="http://public:9000", ) S3StorageBackend(config) calls = mock_boto3_session["session"].return_value.client.call_args_list # First call should use internal endpoint assert calls[0][1]["endpoint_url"] == "http://internal:9000" # Second call should use public endpoint assert calls[1][1]["endpoint_url"] == "http://public:9000" def test_init_uses_internal_for_presign_when_no_public(self, mock_boto3_session): """Test that presign client uses internal endpoint when public is not set.""" config = S3Config( access_key="key", secret_key="secret", bucket_name="bucket", endpoint_url_internal="http://internal:9000", endpoint_url_public=None, ) S3StorageBackend(config) calls = mock_boto3_session["session"].return_value.client.call_args_list # Both should use internal endpoint assert calls[0][1]["endpoint_url"] == "http://internal:9000" assert calls[1][1]["endpoint_url"] == "http://internal:9000" class TestEnsureBucket: """Tests for ensure_bucket method.""" def test_ensure_bucket_creates_bucket_if_not_exists( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that bucket is created if it doesn't exist.""" mock_client = mock_boto3_session["client"] # Simulate bucket not found error_response = {"Error": {"Code": "404"}} mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket") s3_backend.ensure_bucket() mock_client.create_bucket.assert_called_once_with(Bucket="test-bucket") def test_ensure_bucket_creates_bucket_no_such_bucket_error( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that bucket is created on NoSuchBucket error.""" mock_client = mock_boto3_session["client"] error_response = {"Error": {"Code": "NoSuchBucket"}} mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket") s3_backend.ensure_bucket() mock_client.create_bucket.assert_called_once_with(Bucket="test-bucket") def test_ensure_bucket_does_not_create_if_exists( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that bucket is not created if it already exists.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} s3_backend.ensure_bucket() mock_client.create_bucket.assert_not_called() def test_ensure_bucket_caches_result( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that bucket check is cached after first call.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} s3_backend.ensure_bucket() s3_backend.ensure_bucket() s3_backend.ensure_bucket() # Should only check once assert mock_client.head_bucket.call_count == 1 def test_ensure_bucket_raises_on_other_errors( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that other errors are raised.""" mock_client = mock_boto3_session["client"] error_response = {"Error": {"Code": "AccessDenied"}} mock_client.head_bucket.side_effect = ClientError(error_response, "HeadBucket") with pytest.raises(ClientError): s3_backend.ensure_bucket() class TestUploadFileobj: """Tests for upload_fileobj method.""" def test_upload_fileobj_with_content_type( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test uploading a file with content type.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} fileobj = io.BytesIO(b"test content") s3_backend.upload_fileobj("test/key.txt", fileobj, content_type="text/plain") mock_client.upload_fileobj.assert_called_once() call_kwargs = mock_client.upload_fileobj.call_args[1] assert call_kwargs["Bucket"] == "test-bucket" assert call_kwargs["Key"] == "test/key.txt" assert call_kwargs["ExtraArgs"] == {"ContentType": "text/plain"} def test_upload_fileobj_without_content_type( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test uploading a file without content type.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} fileobj = io.BytesIO(b"test content") s3_backend.upload_fileobj("test/key.txt", fileobj, content_type=None) call_kwargs = mock_client.upload_fileobj.call_args[1] assert call_kwargs["ExtraArgs"] is None def test_upload_fileobj_ensures_bucket( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that upload_fileobj calls ensure_bucket.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} fileobj = io.BytesIO(b"test content") s3_backend.upload_fileobj("test/key.txt", fileobj, content_type=None) mock_client.head_bucket.assert_called_once() class TestDownloadFileobj: """Tests for download_fileobj method.""" def test_download_fileobj(self, s3_backend: S3StorageBackend, mock_boto3_session): """Test downloading a file.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} fileobj = io.BytesIO() s3_backend.download_fileobj("test/key.txt", fileobj) mock_client.download_fileobj.assert_called_once_with( "test-bucket", "test/key.txt", fileobj ) def test_download_fileobj_ensures_bucket( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that download_fileobj calls ensure_bucket.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} fileobj = io.BytesIO() s3_backend.download_fileobj("test/key.txt", fileobj) mock_client.head_bucket.assert_called_once() class TestExists: """Tests for exists method.""" def test_exists_returns_true_when_object_exists( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that exists returns True when object exists.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} mock_client.head_object.return_value = {} result = s3_backend.exists("test/key.txt") assert result is True mock_client.head_object.assert_called_once_with( Bucket="test-bucket", Key="test/key.txt" ) def test_exists_returns_false_when_object_not_found_404( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that exists returns False on 404 error.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} error_response = {"Error": {"Code": "404"}} mock_client.head_object.side_effect = ClientError(error_response, "HeadObject") result = s3_backend.exists("test/key.txt") assert result is False def test_exists_returns_false_when_no_such_key( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that exists returns False on NoSuchKey error.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} error_response = {"Error": {"Code": "NoSuchKey"}} mock_client.head_object.side_effect = ClientError(error_response, "HeadObject") result = s3_backend.exists("test/key.txt") assert result is False def test_exists_raises_on_other_errors( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that other errors are raised.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} error_response = {"Error": {"Code": "AccessDenied"}} mock_client.head_object.side_effect = ClientError(error_response, "HeadObject") with pytest.raises(ClientError): s3_backend.exists("test/key.txt") class TestSize: """Tests for size method.""" def test_size_returns_content_length( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that size returns ContentLength from head_object.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} mock_client.head_object.return_value = {"ContentLength": 12345} result = s3_backend.size("test/key.txt") assert result == 12345 mock_client.head_object.assert_called_once_with( Bucket="test-bucket", Key="test/key.txt" ) def test_size_returns_zero_when_no_content_length( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that size returns 0 when ContentLength is missing.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} mock_client.head_object.return_value = {} result = s3_backend.size("test/key.txt") assert result == 0 class TestDelete: """Tests for delete method.""" def test_delete_calls_delete_object( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that delete calls delete_object.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} s3_backend.delete("test/key.txt") mock_client.delete_object.assert_called_once_with( Bucket="test-bucket", Key="test/key.txt" ) def test_delete_ensures_bucket( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that delete calls ensure_bucket.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} s3_backend.delete("test/key.txt") mock_client.head_bucket.assert_called_once() class TestRead: """Tests for read method.""" def test_read_returns_body_content( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that read returns the object body content.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} mock_body = MagicMock() mock_body.read.return_value = b"file content" mock_client.get_object.return_value = {"Body": mock_body} result = s3_backend.read("test/key.txt") assert result == b"file content" mock_client.get_object.assert_called_once_with( Bucket="test-bucket", Key="test/key.txt" ) def test_read_ensures_bucket( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that read calls ensure_bucket.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} mock_body = MagicMock() mock_body.read.return_value = b"content" mock_client.get_object.return_value = {"Body": mock_body} s3_backend.read("test/key.txt") mock_client.head_bucket.assert_called_once() class TestGenerateUrl: """Tests for generate_url method.""" def test_generate_url_returns_presigned_url( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that generate_url returns a presigned URL.""" mock_client = mock_boto3_session["client"] mock_presign_client = mock_boto3_session["presign_client"] mock_client.head_bucket.return_value = {} mock_presign_client.generate_presigned_url.return_value = ( "https://example.com/presigned-url" ) result = s3_backend.generate_url("test/key.txt") assert result == "https://example.com/presigned-url" def test_generate_url_uses_correct_parameters( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test that generate_url uses correct parameters.""" mock_client = mock_boto3_session["client"] mock_presign_client = mock_boto3_session["presign_client"] mock_client.head_bucket.return_value = {} s3_backend.generate_url("test/key.txt") mock_presign_client.generate_presigned_url.assert_called_once_with( ClientMethod="get_object", Params={"Bucket": "test-bucket", "Key": "test/key.txt"}, ExpiresIn=3600, ) def test_generate_url_uses_custom_expiration(self, mock_boto3_session): """Test that generate_url uses custom expiration time.""" config = S3Config( access_key="key", secret_key="secret", bucket_name="bucket", endpoint_url_internal="http://localhost:9000", endpoint_url_public=None, presign_expires_seconds=7200, ) backend = S3StorageBackend(config) mock_client = mock_boto3_session["client"] mock_presign_client = mock_boto3_session["presign_client"] mock_client.head_bucket.return_value = {} backend.generate_url("test/key.txt") call_kwargs = mock_presign_client.generate_presigned_url.call_args[1] assert call_kwargs["ExpiresIn"] == 7200 class TestIntegrationScenarios: """Integration-style tests for common usage scenarios.""" def test_upload_then_check_exists( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test uploading a file and checking it exists.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} mock_client.head_object.return_value = {} # Upload fileobj = io.BytesIO(b"test content") s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain") # Check exists result = s3_backend.exists("test/file.txt") assert result is True def test_upload_then_read_back( self, s3_backend: S3StorageBackend, mock_boto3_session ): """Test uploading a file and reading it back.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} content = b"test file content" mock_body = MagicMock() mock_body.read.return_value = content mock_client.get_object.return_value = {"Body": mock_body} # Upload fileobj = io.BytesIO(content) s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain") # Read back result = s3_backend.read("test/file.txt") assert result == content def test_upload_then_delete(self, s3_backend: S3StorageBackend, mock_boto3_session): """Test uploading a file and then deleting it.""" mock_client = mock_boto3_session["client"] mock_client.head_bucket.return_value = {} # Upload fileobj = io.BytesIO(b"test content") s3_backend.upload_fileobj("test/file.txt", fileobj, content_type="text/plain") # Delete s3_backend.delete("test/file.txt") mock_client.delete_object.assert_called_once_with( Bucket="test-bucket", Key="test/file.txt" ) def test_full_lifecycle(self, s3_backend: S3StorageBackend, mock_boto3_session): """Test full file lifecycle: upload, check, get size, get url, delete.""" mock_client = mock_boto3_session["client"] mock_presign_client = mock_boto3_session["presign_client"] mock_client.head_bucket.return_value = {} mock_client.head_object.return_value = {"ContentLength": 100} mock_presign_client.generate_presigned_url.return_value = "https://url" content = b"test file content" # Upload fileobj = io.BytesIO(content) s3_backend.upload_fileobj("media/file.mp4", fileobj, content_type="video/mp4") # Check exists assert s3_backend.exists("media/file.mp4") is True # Get size assert s3_backend.size("media/file.mp4") == 100 # Generate URL url = s3_backend.generate_url("media/file.mp4") assert url == "https://url" # Delete s3_backend.delete("media/file.mp4") # Verify all operations were called assert mock_client.upload_fileobj.called assert mock_client.delete_object.called