Remove MNP support

Signed-off-by: Francesco De Martino <[email protected]>

Author: demartinofra

Diff for: cli/awsbatch/awsbhosts.py

@@ -228,17 +228,11 @@ def main():
         log = config_logger(args.log_level)
         log.info("Input parameters: %s" % args)
         config = AWSBatchCliConfig(log, args.cluster)
-        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy, endpoint_url=config.endpoint_url,
+        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy,
                                            aws_access_key_id=config.aws_access_key_id,
                                            aws_secret_access_key=config.aws_secret_access_key)
 
-        # FIXME remove
-        if config.endpoint_url:
-            compute_environment = config.compute_environment_mnp
-        else:
-            compute_environment = config.compute_environment
-
-        AWSBhostsCommand(log, boto3_factory).run(compute_environments=[compute_environment],
+        AWSBhostsCommand(log, boto3_factory).run(compute_environments=[config.compute_environment],
                                                  instance_ids=args.instance_ids, show_details=args.details)
 
     except KeyboardInterrupt:

Diff for: cli/awsbatch/awsbjobs.py

@@ -299,7 +299,7 @@ def main():
         log = config_logger(args.log_level)
         log.info("Input parameters: %s" % args)
         config = AWSBatchCliConfig(log=log, cluster=args.cluster)
-        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy, endpoint_url=config.endpoint_url,
+        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy,
                                            aws_access_key_id=config.aws_access_key_id,
                                            aws_secret_access_key=config.aws_secret_access_key)
 
@@ -308,14 +308,9 @@ def main():
         else:
             job_status = ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING', 'RUNNING', 'SUCCEEDED', 'FAILED']
 
-        # FIXME remove
-        if config.endpoint_url:
-            job_queue = config.job_queue_mnp
-        else:
-            job_queue = config.job_queue
-
         AWSBjobsCommand(log, boto3_factory).run(job_status=job_status, expand_arrays=args.expand_arrays,
-                                                job_ids=args.job_ids, job_queue=job_queue, show_details=args.details)
+                                                job_ids=args.job_ids, job_queue=config.job_queue,
+                                                show_details=args.details)
 
     except KeyboardInterrupt:
         print("Exiting...")

Diff for: cli/awsbatch/awsbkill.py

@@ -108,7 +108,7 @@ def main():
         log = config_logger(args.log_level)
         log.info("Input parameters: %s" % args)
         config = AWSBatchCliConfig(log=log, cluster=args.cluster)
-        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy, endpoint_url=config.endpoint_url,
+        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy,
                                            aws_access_key_id=config.aws_access_key_id,
                                            aws_secret_access_key=config.aws_secret_access_key)
         AWSBkillCommand(log, boto3_factory).run(job_ids=args.job_ids)

Diff for: cli/awsbatch/awsbout.py

@@ -191,7 +191,7 @@ def main():
         log = config_logger(args.log_level)
         log.info("Input parameters: %s" % args)
         config = AWSBatchCliConfig(log=log, cluster=args.cluster)
-        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy, endpoint_url=config.endpoint_url,
+        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy,
                                            aws_access_key_id=config.aws_access_key_id,
                                            aws_secret_access_key=config.aws_secret_access_key)
 

Diff for: cli/awsbatch/awsbqueues.py

@@ -126,20 +126,15 @@ def main():
         log = config_logger(args.log_level)
         log.info("Input parameters: %s" % args)
         config = AWSBatchCliConfig(log=log, cluster=args.cluster)
-        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy, endpoint_url=config.endpoint_url,
+        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy,
                                            aws_access_key_id=config.aws_access_key_id,
                                            aws_secret_access_key=config.aws_secret_access_key)
 
         if args.job_queues:
             job_queues = args.job_queues
             show_details = True
         else:
-            # FIXME remove
-            if config.endpoint_url:
-                job_queue = config.job_queue_mnp
-            else:
-                job_queue = config.job_queue
-            job_queues = [job_queue]
+            job_queues = [config.job_queue]
             show_details = args.details
         AWSBqueuesCommand(log, boto3_factory).run(job_queues=job_queues, show_details=show_details)
 

Diff for: cli/awsbatch/awsbsub.py

@@ -62,9 +62,6 @@ def _get_parser():
     parser.add_argument('-t', '--timeout', help='The time duration in seconds (measured from the job attempt\'s '
                                                 'startedAt timestamp) after which AWS Batch terminates your jobs '
                                                 'if they have not finished. It must be at least 60 seconds', type=int)
-    # MNP parameter
-    parser.add_argument('-n', '--nodes', help='The number of nodes to reserve for the job. '
-                                              'It enables Multi Node Parallel submission', type=int)
     # array parameters
     parser.add_argument('-a', '--array-size', help='The size of the array. It can be between 2 and 10,000. '
                                                    'If you specify array properties for a job, '
@@ -370,7 +367,7 @@ def main():
         log = config_logger(args.log_level)
         log.info("Input parameters: %s" % args)
         config = AWSBatchCliConfig(log=log, cluster=args.cluster)
-        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy, endpoint_url=config.endpoint_url,
+        boto3_factory = Boto3ClientFactory(region=config.region, proxy=config.proxy,
                                            aws_access_key_id=config.aws_access_key_id,
                                            aws_secret_access_key=config.aws_secret_access_key)
 
@@ -392,20 +389,10 @@ def main():
         # parse and validate depends_on parameter
         depends_on = _get_depends_on(args)
 
-        # select submission (standard vs MNP)
-        if args.nodes or config.endpoint_url:  # FIXME remove config.endpoint from if
-            job_definition = config.job_definition_mnp
-        else:
-            job_definition = config.job_definition
-
-        # FIXME remove
-        if config.endpoint_url:
-            job_queue = config.job_queue_mnp
-        else:
-            job_queue = config.job_queue
+        job_definition = config.job_definition
 
         AWSBsubCommand(log, boto3_factory).run(job_definition=job_definition, job_name=job_name,
-                                               job_queue=job_queue, command=command, nodes=args.nodes,
+                                               job_queue=config.job_queue, command=command,
                                                vcpus=args.vcpus, memory=args.memory,
                                                array_size=args.array_size, dependencies=depends_on,
                                                retry_attempts=args.retry_attempts, timeout=args.timeout,

Diff for: cli/awsbatch/common.py

@@ -85,32 +85,22 @@ class Boto3ClientFactory(object):
     """
     Boto3 configuration object
     """
-    def __init__(self, region, aws_access_key_id, aws_secret_access_key, proxy='NONE', endpoint_url=None):
+    def __init__(self, region, aws_access_key_id, aws_secret_access_key, proxy='NONE'):
         self.region = region
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key
         self.proxy_config = Config()
-        self.endpoint_url = endpoint_url
         if not proxy == 'NONE':
             self.proxy_config = Config(proxies={'https': proxy})
 
     def get_client(self, service):
         try:
-            if service == 'batch' and self.endpoint_url is not None:
-                # FIXME remove this ack used for testing
-                boto3_service = 'batch-zeta'
-                boto3_endpoint_url = self.endpoint_url
-            else:
-                boto3_service = service
-                boto3_endpoint_url = None
-
-            return boto3.client(boto3_service, region_name=self.region,
-                                endpoint_url=boto3_endpoint_url,
+            return boto3.client(service, region_name=self.region,
                                 aws_access_key_id=self.aws_access_key_id,
                                 aws_secret_access_key=self.aws_secret_access_key,
                                 config=self.proxy_config)
         except ClientError as e:
-            fail("AWS %s service failed with exception: %s" % (boto3_service, e))
+            fail("AWS %s service failed with exception: %s" % (service, e))
 
 
 class AWSBatchCliConfig(object):
@@ -128,7 +118,6 @@ def __init__(self, log, cluster):
         self.aws_access_key_id = None
         self.aws_secret_access_key = None
         self.region = None
-        self.endpoint_url = None
         cfncluster_config_file = os.path.expanduser(os.path.join('~', '.cfncluster', 'config'))
         if os.path.isfile(cfncluster_config_file):
             self.__init_from_cfncluster_config(cfncluster_config_file, log)
@@ -157,7 +146,6 @@ def __verify_initialization(self, log):
             log.debug("compute_environment = %s", self.compute_environment)
             log.debug("job_queue = %s", self.job_queue)
             log.debug("job_definition = %s", self.job_definition)
-            # FIXME restore log.debug("job_definition_mnp = %s", self.job_definition_mnp)
             log.debug("master_ip = %s", self.master_ip)
             log.info(self)
         except AttributeError as e:
@@ -213,10 +201,6 @@ def __init_from_config(self, cli_config_file, cluster, log):
                 self.region = config.get('main', 'region')
             except NoOptionError:
                 pass
-            try:
-                self.endpoint_url = config.get('main', 'endpoint_url')
-            except NoOptionError:
-                pass
 
             try:
                 self.stack_name = 'cfncluster-' + cluster_name
@@ -228,7 +212,6 @@ def __init_from_config(self, cli_config_file, cluster, log):
                 self.compute_environment = config.get(cluster_section, 'compute_environment')
                 self.job_queue = config.get(cluster_section, 'job_queue')
                 self.job_definition = config.get(cluster_section, 'job_definition')
-                self.job_definition_mnp = config.get(cluster_section, 'job_definition_mnp')
                 self.master_ip = config.get(cluster_section, 'master_ip')
 
                 # get proxy
@@ -272,16 +255,10 @@ def __init_from_stack(self, cluster, log):
                         self.s3_bucket = output_value
                     elif output_key == 'BatchComputeEnvironmentArn':
                         self.compute_environment = output_value
-                    elif output_key == 'MNPComputeEnvironmentArn':  # FIXME remove
-                        self.compute_environment_mnp = output_value
                     elif output_key == 'BatchJobQueueArn':
                         self.job_queue = output_value
-                    elif output_key == 'MNPJobQueueArn':  # FIXME remove
-                        self.job_queue_mnp = output_value
                     elif output_key == 'BatchJobDefinitionArn':
                         self.job_definition = output_value
-                    elif output_key == 'MNPJobDefinitionArn':  # FIXME remove
-                        self.job_definition_mnp = output_value
                     elif output_key == 'MasterPrivateIP':
                         self.master_ip = output_value
 

Diff for: cli/awsbatch/examples/awsbatch-cli.cfg

@@ -33,8 +33,6 @@ compute_environment = arn:aws:batch:<region>:<account-id>:compute-environment/cf
 job_queue = arn:aws:batch:<region>:<account-id>:job-queue/cfncluster-<cluster-name>
 # Job Definition ARN used for standard job submission
 job_definition = arn:aws:batch:<region>:<account-id>:job-definition/cfncluster-<cluster-name>:1
-# Job Definition ARN used for Multi Node Parallel job submission
-job_definition_mnp = arn:aws:batch:<region>:<account-id>:job-definition/cfncluster-<cluster-name>-mnp:1
 # HTTP(S) proxy server, typically http://x.x.x.x:8080, used for internal boto3 calls
 proxy = NONE
 # Private Master IP, used internally in the job submission phase.