import dataclasses import datetime import functools import json import logging import time import typing import uuid import boto3 import botocore import requests from botocore.client import BaseClient from botocore.errorfactory import BaseClientExceptions from botocore.exceptions import ClientError logger = logging.getLogger(__name__) # fmt: off _response_describe_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '38d7b555-2699-4868-9139-c47b1c2d70db', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '38d7b555-2699-4868-9139-c47b1c2d70db', 'content-type': 'application/json', 'content-length': '2314', 'date': 'Tue, 11 Jan 2022 07:52:24 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}} _response_list_domain_names_success = {'ResponseMetadata': {'RequestId': '25923514-a426-4223-939a-4436f1d6e243', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '25923514-a426-4223-939a-4436f1d6e243', 'content-type': 'application/json', 'content-length': '87', 'date': 'Tue, 11 Jan 2022 07:53:37 GMT'}, 'RetryAttempts': 0}, 'DomainNames': [{'DomainName': 'es-d8a24850a0ae4b82af26', 'EngineType': 'Elasticsearch'}]} _response_describe_elasticsearch_domains_success = {'ResponseMetadata': {'RequestId': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '61cfe5f4-42e7-4233-bca9-53f0ca62099d', 'content-type': 'application/json', 'content-length': '2320', 'date': 'Tue, 11 Jan 2022 07:55:25 GMT'}, 'RetryAttempts': 0}, 'DomainStatusList': [{'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': False, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': False, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}]} _response_delete_elasticsearch_domain_not_found = ClientError({'Error': {'Message': 'Domain not found: asd', 'Code': 'ResourceNotFoundException'}, 'ResponseMetadata': {'RequestId': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'HTTPStatusCode': 409, 'HTTPHeaders': {'x-amzn-requestid': 'd0669560-77f6-424c-bbc6-fb5878daa494', 'x-amzn-errortype': 'ResourceNotFoundException', 'content-type': 'application/json', 'content-length': '35', 'date': 'Tue, 11 Jan 2022 08:06:07 GMT'}, 'RetryAttempts': 0}}, 'DeleteElasticsearchDomain') _response_delete_elasticsearch_domain_success = {'ResponseMetadata': {'RequestId': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '1e9046f4-c4ee-4fda-8dfe-dd23f0161b7e', 'content-type': 'application/json', 'content-length': '2312', 'date': 'Tue, 11 Jan 2022 08:31:33 GMT'}, 'RetryAttempts': 0}, 'DomainStatus': {'DomainId': '400344683105/es-d8a24850a0ae4b82af26', 'DomainName': 'es-d8a24850a0ae4b82af26', 'ARN': 'arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', 'Created': True, 'Deleted': True, 'Endpoints': {'vpc': 'vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com'}, 'Processing': True, 'UpgradeProcessing': False, 'ElasticsearchVersion': '7.8', 'ElasticsearchClusterConfig': {'InstanceType': 't2.medium.elasticsearch', 'InstanceCount': 2, 'DedicatedMasterEnabled': False, 'ZoneAwarenessEnabled': True, 'ZoneAwarenessConfig': {'AvailabilityZoneCount': 2}, 'WarmEnabled': False, 'ColdStorageOptions': {'Enabled': False}}, 'EBSOptions': {'EBSEnabled': True, 'VolumeType': 'gp2', 'VolumeSize': 10}, 'AccessPolicies': '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"es:*","Resource":"*"}]}', 'SnapshotOptions': {'AutomatedSnapshotStartHour': 0}, 'VPCOptions': {'VPCId': 'vpc-8c28b5e7', 'SubnetIds': ['subnet-004909d95a03c5fcd', 'subnet-b8f596f5'], 'AvailabilityZones': ['eu-central-1a', 'eu-central-1c'], 'SecurityGroupIds': ['sg-c11a77ac']}, 'CognitoOptions': {'Enabled': False}, 'EncryptionAtRestOptions': {'Enabled': False}, 'NodeToNodeEncryptionOptions': {'Enabled': False}, 'AdvancedOptions': {'override_main_response_version': 'false', 'rest.action.multi.allow_explicit_index': 'true'}, 'ServiceSoftwareOptions': {'CurrentVersion': 'R20211203-P2', 'NewVersion': '', 'UpdateAvailable': False, 'Cancellable': False, 'UpdateStatus': 'COMPLETED', 'Description': 'There is no software update available for this domain.', 'AutomatedUpdateDate': datetime.datetime(2021, 12, 14, 3, 38, 38), 'OptionalDeployment': False}, 'DomainEndpointOptions': {'EnforceHTTPS': True, 'TLSSecurityPolicy': 'Policy-Min-TLS-1-0-2019-07', 'CustomEndpointEnabled': False}, 'AdvancedSecurityOptions': {'Enabled': False, 'InternalUserDatabaseEnabled': False}, 'AutoTuneOptions': {'State': 'ENABLE_IN_PROGRESS'}}} # fmt: on def retry(timeout: int, backoff_seconds: int = 10, bubble_errors: typing.List[typing.Type[Exception]] = None): """ Tries calling a function {timeout} seconds until it succeeds or gives up and throw an error :param timeout: Timeout in seconds :param backoff_seconds: Wait time between retries :param bubble_errors: List of exceptions to bubble up :return: Wrapped function """ bubble_errors = tuple(bubble_errors or []) def wrapped(func: typing.Callable): @functools.wraps(func) def func_retrier(*args, **kwargs): waited = 0 started_at = datetime.datetime.now() while True: try: logger.debug(f"Calling {func}") result = func(*args, **kwargs) logger.debug(f"{func} completed in {(datetime.datetime.now() - started_at)}") return result except KeyboardInterrupt: raise except tuple(bubble_errors): raise except Exception as e: doze = min(timeout - waited, backoff_seconds) times = (timeout - waited) // backoff_seconds logger.debug(f"{func} failed, will try {times} times in {doze} sec", exc_info=True) time.sleep(doze) waited += backoff_seconds if waited >= timeout: raise TimeoutError(f"Couldn't get a successful result from {func} in {timeout} seconds") from e return func_retrier return wrapped @dataclasses.dataclass class ElasticsearchInstance: domain_name: str endpoint: str arn: str class ElasticsearchService: domain_name_format = "es-{}" default_version = "7.8" valid_versions = {"7.10", "7.9", "7.8", "7.7", "7.4", "7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6", "5.5"} # fmt: skip def __init__(self, region: str): self.es = boto3.client("es", region_name=region) def create_instance( self, node_type: str, total_nodes: int, subnet_group_ids: typing.List[str], security_group_id: str, version: str = None, tags: dict = None, volume_size_gb: int = 30, total_availability_zones: int = 3, ) -> ElasticsearchInstance: if not version: version = self.default_version assert version in self.valid_versions, "Invalid version" domain_name = self.domain_name_format.format(uuid.uuid4().hex[:20]) logger.info( "Creating a new Elasticsearch domain", extra=dict(version=version, domain_name=domain_name, node_type=node_type, total_nodes=total_nodes), ) es_result = self.es.create_elasticsearch_domain( DomainName=domain_name, ElasticsearchVersion=version, ElasticsearchClusterConfig={ "InstanceType": node_type, "InstanceCount": total_nodes, "ZoneAwarenessEnabled": True, "ZoneAwarenessConfig": { "AvailabilityZoneCount": total_availability_zones, }, }, VPCOptions={ "SubnetIds": subnet_group_ids, "SecurityGroupIds": [security_group_id], }, EBSOptions={ "EBSEnabled": True, "VolumeType": "gp2", "VolumeSize": volume_size_gb, }, AccessPolicies=json.dumps( { "Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": "*", "Action": "es:*", "Resource": "*"}], } ), DomainEndpointOptions={"EnforceHTTPS": True}, TagList=[{"Key": k, "Value": v} for k, v in (tags or {}).items()], ) logger.info("Waiting until ES domain becomes available. This will take a while (~15 min)") return self._find_instance(domain_name) @retry(timeout=20 * 60, backoff_seconds=30) def _find_instance(self, domain_name: str) -> ElasticsearchInstance: result = self.es.describe_elasticsearch_domain(DomainName=domain_name)["DomainStatus"] return ElasticsearchInstance(endpoint=result["Endpoints"]["vpc"], arn=result["ARN"], domain_name=domain_name) def find_domain_name_by_arn(self, arn: str): res = self.es.list_domain_names(EngineType='Elasticsearch') domain_names = [it['DomainName'] for it in res['DomainNames']] res = self.es.describe_elasticsearch_domains(DomainNames=domain_names) domain_name_by_arn = {it['ARN']: it['DomainName'] for it in res['DomainStatusList']} print(res) def main(): username, password = 'bgNNs4EWPTsuQJ24', '3asinFjikT5MgwRz!' """ GET _search { "query": { "match_all": {} } } ### PUT /my-index PUT /my-index2 PUT /my-index3 PUT /my-index3-4 ### DELETE /my-index*?expand_wildcards=all ### DELETE /my-index1 ### GET /my-index GET /_stats/indexing ### GET /_cat/indices?v ### GET /kibana_sample_data_ecommerce/ ### GET /_all/_mapping ## delete multiple PUT /my-index PUT /my-index2 DELETE /my-index,my-index2 GET /my-index ### GET /_aliases ### GET /_stats/ user: qdSmWKS8p3VrUyBC pwd: un6f2vTrRjoa8orn! """ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format=f"%(asctime)s: {logging.BASIC_FORMAT}") logging.getLogger("botocore").setLevel(logging.INFO) # ElasticsearchService('eu-central-1') e = ElasticsearchService("eu-central-1") esi = ElasticsearchInstance( endpoint='vpc-es-d8a24850a0ae4b82af26-gcw46l5kiiy6oylbhpqpr7s454.eu-central-1.es.amazonaws.com', arn='arn:aws:es:eu-central-1:400344683105:domain/es-d8a24850a0ae4b82af26', domain_name='es-d8a24850a0ae4b82af26', ) # e.find_instance_by_arn(esi.arn) # exit() try: es = boto3.client('es') res = es.delete_elasticsearch_domain(DomainName=esi.domain_name) print(res) except ClientError as e: error_code = e.__class__.__name__ print(error_code) e = ElasticsearchService("eu-central-1") # print( # e.create_instance( # node_type="t2.medium.elasticsearch", # total_nodes=2, # volume_size_gb=10, # total_availability_zones=2, # security_group_id="sg-c11a77ac", # subnet_group_ids=["subnet-b8f596f5", "subnet-004909d95a03c5fcd"], # tags={"CostCenter": "my"}, # ) # )