unthaw S3 objects (#3406)
This commit is contained in:
@@ -40,12 +40,14 @@ class AsyncAWSClient:
|
|||||||
aws_secret_access_key: str | None = None,
|
aws_secret_access_key: str | None = None,
|
||||||
region_name: str | None = None,
|
region_name: str | None = None,
|
||||||
endpoint_url: str | None = None,
|
endpoint_url: str | None = None,
|
||||||
|
profile_name: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.region_name = region_name or settings.AWS_REGION
|
self.region_name = region_name or settings.AWS_REGION
|
||||||
self._endpoint_url = endpoint_url
|
self._endpoint_url = endpoint_url
|
||||||
self.session = aioboto3.Session(
|
self.session = aioboto3.Session(
|
||||||
aws_access_key_id=aws_access_key_id,
|
aws_access_key_id=aws_access_key_id,
|
||||||
aws_secret_access_key=aws_secret_access_key,
|
aws_secret_access_key=aws_secret_access_key,
|
||||||
|
profile_name=profile_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _ecs_client(self) -> ECSClient:
|
def _ecs_client(self) -> ECSClient:
|
||||||
@@ -276,6 +278,65 @@ class AsyncAWSClient:
|
|||||||
object_keys.append(obj["Key"])
|
object_keys.append(obj["Key"])
|
||||||
return object_keys
|
return object_keys
|
||||||
|
|
||||||
|
async def delete_files(self, bucket: str, keys: list[str]) -> None:
|
||||||
|
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/delete_objects.html
|
||||||
|
"""
|
||||||
|
Delete multiple objects from S3 bucket.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
bucket: The S3 bucket name
|
||||||
|
keys: List of object keys to delete
|
||||||
|
"""
|
||||||
|
if not keys:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with self._s3_client() as client:
|
||||||
|
# Format the objects for the delete_objects call
|
||||||
|
objects = [{"Key": key} for key in keys]
|
||||||
|
|
||||||
|
response = await client.delete_objects(
|
||||||
|
Bucket=bucket,
|
||||||
|
Delete={
|
||||||
|
"Objects": objects,
|
||||||
|
"Quiet": False, # Set to True to suppress response details
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Log any errors that occurred during deletion
|
||||||
|
if "Errors" in response:
|
||||||
|
for error in response["Errors"]:
|
||||||
|
LOG.error(
|
||||||
|
"Failed to delete object from S3",
|
||||||
|
bucket=bucket,
|
||||||
|
key=error.get("Key"),
|
||||||
|
code=error.get("Code"),
|
||||||
|
message=error.get("Message"),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
LOG.exception("Failed to delete files from S3", bucket=bucket, keys_count=len(keys))
|
||||||
|
raise e
|
||||||
|
|
||||||
|
async def restore_object(self, bucket: str, key: str, days: int = 1, tier: str = "Standard") -> None:
|
||||||
|
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/restore_object.html
|
||||||
|
"""
|
||||||
|
Restore an archived S3 object from GLACIER storage class.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
bucket: The S3 bucket name
|
||||||
|
key: The S3 object key
|
||||||
|
days: Number of days to keep the restored object available (default: 1)
|
||||||
|
tier: Restoration tier - "Standard" (3-5 hours) or "Expedited" (1-5 minutes)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
async with self._s3_client() as client:
|
||||||
|
await client.restore_object(
|
||||||
|
Bucket=bucket, Key=key, RestoreRequest={"Days": days, "GlacierJobParameters": {"Tier": tier}}
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
LOG.exception("Failed to restore S3 object", bucket=bucket, key=key, tier=tier)
|
||||||
|
raise e
|
||||||
|
|
||||||
async def run_task(
|
async def run_task(
|
||||||
self,
|
self,
|
||||||
cluster: str,
|
cluster: str,
|
||||||
|
|||||||
Reference in New Issue
Block a user