initial commit
This commit is contained in:
@@ -0,0 +1,238 @@
|
||||
import dataclasses
|
||||
import datetime
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
import botocore
|
||||
import requests
|
||||
from botocore.client import BaseClient
|
||||
from botocore.errorfactory import BaseClientExceptions
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# fmt: off
|
||||
_response_describe_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '38d7b555-2699-4868-9139-c47b1c2d70db', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '38d7b555-2699-4868-9139-c47b1c2d70db', 'content-type': 'application/json', 'content-length': '2314', 'date': 'Tue, 11 Jan 2022 07:52:24 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}}
|
||||
_response_list_domain_names_success = {'ResponseMetadata': {'RequestId': '25923514-a426-4223-939a-4436f1d6e243', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '25923514-a426-4223-939a-4436f1d6e243', 'content-type': 'application/json', 'content-length': '87', 'date': 'Tue, 11 Jan 2022 07:53:37 GMT'}, 'RetryAttempts': 0}, 'DomainNames': [{'DomainName': 'es-d8a24850a0ae4b82af26', 'EngineType': 'Elasticsearch'}]}
|
||||
_response_describe_elasticsearch_domains_success = {'ResponseMetadata': {'RequestId': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'content-type': 'application/json', 'content-length': '2320', 'date': 'Tue, 11 Jan 2022 07:55:25 GMT'}, 'RetryAttempts': 0}, 'DomainStatusList': [{'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}]}
|
||||
_response_delete_elasticsearch_domain_not_found = ClientError({'Error': {'Message': 'Domain not found: asd', 'Code': 'ResourceNotFoundException'}, 'ResponseMetadata': {'RequestId': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'HTTPStatusCode': 409, 'HTTPHeaders': {'x-amzn-requestid': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'x-amzn-errortype': 'ResourceNotFoundException', 'content-type': 'application/json', 'content-length': '35', 'date': 'Tue, 11 Jan 2022 08:06:07 GMT'}, 'RetryAttempts': 0}}, 'DeleteElasticsearchDomain')
|
||||
_response_delete_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'content-type': 'application/json', 'content-length': '2312', 'date': 'Tue, 11 Jan 2022 08:31:33 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': True, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': True, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}}
|
||||
# fmt: on
|
||||
|
||||
|
||||
def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None):
|
||||
"""
|
||||
Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error
|
||||
:param timeout: Timeout in seconds
|
||||
:param backoff_seconds: Wait time between retries
|
||||
:param bubble_errors: List of exceptions to bubble up
|
||||
:return: Wrapped function
|
||||
"""
|
||||
bubble_errors = tuple(bubble_errors or [])
|
||||
|
||||
def wrapped(func: typing.Callable):
|
||||
@functools.wraps(func)
|
||||
def func_retrier(*args, **kwargs):
|
||||
waited = 0
|
||||
started_at = datetime.datetime.now()
|
||||
while True:
|
||||
try:
|
||||
logger.debug(f"Calling {func}")
|
||||
result = func(*args, **kwargs)
|
||||
logger.debug(f"{func} completed in {(datetime.datetime.now() - started_at)}")
|
||||
return result
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except tuple(bubble_errors):
|
||||
raise
|
||||
except Exception as e:
|
||||
doze = min(timeout - waited, backoff_seconds)
|
||||
times = (timeout - waited) // backoff_seconds
|
||||
logger.debug(f"{func} failed, will try {times} times in {doze} sec", exc_info=True)
|
||||
time.sleep(doze)
|
||||
waited += backoff_seconds
|
||||
if waited >= timeout:
|
||||
raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e
|
||||
|
||||
return func_retrier
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ElasticsearchInstance:
|
||||
domain_name: str
|
||||
endpoint: str
|
||||
arn: str
|
||||
|
||||
|
||||
class ElasticsearchService:
|
||||
domain_name_format = "es-{}"
|
||||
default_version = "7.8"
|
||||
valid_versions = {"7.10", "7.9", "7.8", "7.7", "7.4", "7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6", "5.5"} # fmt: skip
|
||||
|
||||
def __init__(self, region: str):
|
||||
self.es = boto3.client("es", region_name=region)
|
||||
|
||||
def create_instance(
|
||||
self,
|
||||
node_type: str,
|
||||
total_nodes: int,
|
||||
subnet_group_ids: typing.List[str],
|
||||
security_group_id: str,
|
||||
version: str = None,
|
||||
tags: dict = None,
|
||||
volume_size_gb: int = 30,
|
||||
total_availability_zones: int = 3,
|
||||
) -> ElasticsearchInstance:
|
||||
if not version:
|
||||
version = self.default_version
|
||||
assert version in self.valid_versions, "Invalid version"
|
||||
|
||||
domain_name = self.domain_name_format.format(uuid.uuid4().hex[:20])
|
||||
logger.info(
|
||||
"Creating a new Elasticsearch domain",
|
||||
extra=dict(version=version, domain_name=domain_name, node_type=node_type, total_nodes=total_nodes),
|
||||
)
|
||||
es_result = self.es.create_elasticsearch_domain(
|
||||
DomainName=domain_name,
|
||||
ElasticsearchVersion=version,
|
||||
ElasticsearchClusterConfig={
|
||||
"InstanceType": node_type,
|
||||
"InstanceCount": total_nodes,
|
||||
"ZoneAwarenessEnabled": True,
|
||||
"ZoneAwarenessConfig": {
|
||||
"AvailabilityZoneCount": total_availability_zones,
|
||||
},
|
||||
},
|
||||
VPCOptions={
|
||||
"SubnetIds": subnet_group_ids,
|
||||
"SecurityGroupIds": [security_group_id],
|
||||
},
|
||||
EBSOptions={
|
||||
"EBSEnabled": True,
|
||||
"VolumeType": "gp2",
|
||||
"VolumeSize": volume_size_gb,
|
||||
},
|
||||
AccessPolicies=json.dumps(
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{"Effect": "Allow", "Principal": "*", "Action": "es:*", "Resource": "*"}],
|
||||
}
|
||||
),
|
||||
DomainEndpointOptions={"EnforceHTTPS": True},
|
||||
TagList=[{"Key": k, "Value": v} for k, v in (tags or {}).items()],
|
||||
)
|
||||
logger.info("Waiting until ES domain becomes available. This will take a while (~15 min)")
|
||||
return self._find_instance(domain_name)
|
||||
|
||||
@retry(timeout=20 * 60, backoff_seconds=30)
|
||||
def _find_instance(self, domain_name: str) -> ElasticsearchInstance:
|
||||
result = self.es.describe_elasticsearch_domain(DomainName=domain_name)["DomainStatus"]
|
||||
return ElasticsearchInstance(endpoint=result["Endpoints"]["vpc"], arn=result["ARN"], domain_name=domain_name)
|
||||
|
||||
def find_domain_name_by_arn(self, arn: str):
|
||||
res = self.es.list_domain_names(EngineType='Elasticsearch')
|
||||
domain_names = [it['DomainName'] for it in res['DomainNames']]
|
||||
|
||||
res = self.es.describe_elasticsearch_domains(DomainNames=domain_names)
|
||||
domain_name_by_arn = {it['ARN']: it['DomainName'] for it in res['DomainStatusList']}
|
||||
|
||||
print(res)
|
||||
|
||||
|
||||
def main():
|
||||
username, password = 'bgNNs4EWPTsuQJ24', '3asinFjikT5MgwRz!'
|
||||
|
||||
|
||||
"""
|
||||
GET _search
|
||||
{
|
||||
"query": {
|
||||
"match_all": {}
|
||||
}
|
||||
}
|
||||
|
||||
###
|
||||
PUT /my-index
|
||||
PUT /my-index2
|
||||
PUT /my-index3
|
||||
PUT /my-index3-4
|
||||
|
||||
###
|
||||
DELETE /my-index*?expand_wildcards=all
|
||||
|
||||
###
|
||||
DELETE /my-index1
|
||||
|
||||
###
|
||||
GET /my-index
|
||||
GET /_stats/indexing
|
||||
###
|
||||
GET /_cat/indices?v
|
||||
|
||||
###
|
||||
|
||||
GET /kibana_sample_data_ecommerce/
|
||||
|
||||
###
|
||||
GET /_all/_mapping
|
||||
|
||||
## delete multiple
|
||||
PUT /my-index
|
||||
PUT /my-index2
|
||||
DELETE /my-index,my-index2
|
||||
GET /my-index
|
||||
|
||||
###
|
||||
GET /_aliases
|
||||
|
||||
###
|
||||
GET /_stats/
|
||||
|
||||
|
||||
user: qdSmWKS8p3VrUyBC
|
||||
pwd: un6f2vTrRjoa8orn!
|
||||
"""
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}")
|
||||
logging.getLogger("botocore").setLevel(logging.INFO)
|
||||
|
||||
# ElasticsearchService('eu-central-1')
|
||||
e = ElasticsearchService("eu-central-1")
|
||||
|
||||
esi = ElasticsearchInstance(
|
||||
endpoint='vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com',
|
||||
arn='arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26',
|
||||
domain_name='es-d8a24850a0ae4b82af26',
|
||||
)
|
||||
|
||||
# e.find_instance_by_arn(esi.arn)
|
||||
# exit()
|
||||
try:
|
||||
es = boto3.client('es')
|
||||
res = es.delete_elasticsearch_domain(DomainName=esi.domain_name)
|
||||
print(res)
|
||||
except ClientError as e:
|
||||
error_code = e.__class__.__name__
|
||||
print(error_code)
|
||||
|
||||
e = ElasticsearchService("eu-central-1")
|
||||
# print(
|
||||
# e.create_instance(
|
||||
# node_type="t2.medium.elasticsearch",
|
||||
# total_nodes=2,
|
||||
# volume_size_gb=10,
|
||||
# total_availability_zones=2,
|
||||
# security_group_id="sg-c11a77ac",
|
||||
# subnet_group_ids=["subnet-b8f596f5", "subnet-004909d95a03c5fcd"],
|
||||
# tags={"CostCenter": "my"},
|
||||
# )
|
||||
# )
|
||||
Reference in New Issue
Block a user