diff --git a/.gitignore b/.gitignore index 3d178992123d..3e3728dfc137 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.ipr *.iml *.iws +*.pyc .idea/ sbt/*.jar .settings diff --git a/LICENSE b/LICENSE index 1c1c2c0255fa..aa5d05c4ca33 100644 --- a/LICENSE +++ b/LICENSE @@ -236,6 +236,54 @@ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================= +For the moto library (ec2/third_party/moto*.zip) +======================================================================= + +Copyright 2012 Steve Pulec + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +======================================================================= +For the mock library (ec2/third_party/mock*.zip) +======================================================================= + +Copyright (c) 2003-2012, Michael Foord +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ======================================================================== For CloudPickle (pyspark/cloudpickle.py): diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py index d8840c94ac17..88807feb4a44 100755 --- a/ec2/spark_ec2.py +++ b/ec2/spark_ec2.py @@ -37,6 +37,8 @@ from boto.ec2.blockdevicemapping import BlockDeviceMapping, EBSBlockDeviceType from boto import ec2 +AWS_EVENTUAL_CONSISTENCY = 30 + class UsageError(Exception): pass @@ -223,7 +225,7 @@ def launch_cluster(conn, opts, cluster_name): sys.exit(1) if opts.key_pair is None: print >> stderr, "ERROR: Must provide a key pair name (-k) to use on instances." - sys.exit(1) + sys.exit(1) print "Setting up security groups..." master_group = get_or_make_group(conn, cluster_name + "-master") slave_group = get_or_make_group(conn, cluster_name + "-slaves") @@ -459,7 +461,6 @@ def setup_spark_cluster(master, opts): if opts.ganglia: print "Ganglia started at http://%s:5080/ganglia" % master - # Wait for a whole cluster (masters, slaves and ZooKeeper) to start up def wait_for_cluster(conn, wait_secs, master_nodes, slave_nodes): print "Waiting for instances to start up..." @@ -665,6 +666,59 @@ def get_partition(total, num_partitions, current_partitions): return num_slaves_this_zone +def destroy_cluster(conn, opts, cluster_name): + (master_nodes, slave_nodes) = get_existing_cluster( + conn, opts, cluster_name, die_on_error=False) + print "Terminating master..." + for inst in master_nodes: + inst.terminate() + print "Terminating slaves..." + for inst in slave_nodes: + inst.terminate() + + # Delete security groups as well + if opts.delete_groups: + print "Deleting security groups (this will take some time)..." + group_names = [cluster_name + "-master", cluster_name + "-slaves"] + + attempt = 1; + while attempt <= 3: + print "Attempt %d" % attempt + groups = [g for g in conn.get_all_security_groups() if g.name in group_names] + success = True + # Delete individual rules in all groups before deleting groups to + # remove dependencies between them + for group in groups: + print "Deleting rules in security group " + group.name + for rule in group.rules: + for grant in rule.grants: + success &= group.revoke(ip_protocol=rule.ip_protocol, + from_port=rule.from_port, + to_port=rule.to_port, + src_group=grant) + + # Sleep for AWS eventual-consistency to catch up, and for instances + # to terminate + time.sleep(AWS_EVENTUAL_CONSISTENCY) # Yes, it does have to be this long :-( + for group in groups: + try: + conn.delete_security_group(group.name) + print "Deleted security group " + group.name + except boto.exception.EC2ResponseError: + success = False; + print "Failed to delete security group " + group.name + + # Unfortunately, group.revoke() returns True even if a rule was not + # deleted, so this needs to be rerun if something fails + if success: break; + + attempt += 1 + + if not success: + print "Failed to delete all security groups after 3 tries." + print "Try re-running in a few minutes." + + def real_main(): (opts, action, cluster_name) = parse_args() try: @@ -695,56 +749,7 @@ def real_main(): cluster_name + "?\nALL DATA ON ALL NODES WILL BE LOST!!\n" + "Destroy cluster " + cluster_name + " (y/N): ") if response == "y": - (master_nodes, slave_nodes) = get_existing_cluster( - conn, opts, cluster_name, die_on_error=False) - print "Terminating master..." - for inst in master_nodes: - inst.terminate() - print "Terminating slaves..." - for inst in slave_nodes: - inst.terminate() - - # Delete security groups as well - if opts.delete_groups: - print "Deleting security groups (this will take some time)..." - group_names = [cluster_name + "-master", cluster_name + "-slaves"] - - attempt = 1; - while attempt <= 3: - print "Attempt %d" % attempt - groups = [g for g in conn.get_all_security_groups() if g.name in group_names] - success = True - # Delete individual rules in all groups before deleting groups to - # remove dependencies between them - for group in groups: - print "Deleting rules in security group " + group.name - for rule in group.rules: - for grant in rule.grants: - success &= group.revoke(ip_protocol=rule.ip_protocol, - from_port=rule.from_port, - to_port=rule.to_port, - src_group=grant) - - # Sleep for AWS eventual-consistency to catch up, and for instances - # to terminate - time.sleep(30) # Yes, it does have to be this long :-( - for group in groups: - try: - conn.delete_security_group(group.name) - print "Deleted security group " + group.name - except boto.exception.EC2ResponseError: - success = False; - print "Failed to delete security group " + group.name - - # Unfortunately, group.revoke() returns True even if a rule was not - # deleted, so this needs to be rerun if something fails - if success: break; - - attempt += 1 - - if not success: - print "Failed to delete all security groups after 3 tries." - print "Try re-running in a few minutes." + destroy_cluster() elif action == "login": (master_nodes, slave_nodes) = get_existing_cluster( diff --git a/ec2/tests.py b/ec2/tests.py new file mode 100644 index 000000000000..21fa9dd2a1f8 --- /dev/null +++ b/ec2/tests.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Module: tests.py + +This module holds all unit tests defined for the deployment to EC2 +functionality of Spark. Run this module directly to execute the tests: + + $ python tests.py +""" +import unittest + +from boto import ec2 +import mock +import moto + +import spark_ec2 + + +class CommandTests(unittest.TestCase): + """ + CommandTests defines unit tests for the commands that can be passed to the + deployment script: launch, destroy, login, etc ... + """ + + @moto.mock_ec2 + def test_destroy(self): + spark_ec2.AWS_EVENTUAL_CONSISTENCY = 1 + opts = mock.MagicMock(name='opts') + opts.region = "us-east-1" + conn = ec2.connect_to_region(opts.region) + cluster_name = "cluster_name" + try: + spark_ec2.destroy_cluster(conn, opts, cluster_name) + except: + self.fail("destroy_cluster raised unexpected exception") + +if __name__ == '__main__': + unittest.main() diff --git a/ec2/third_party/mock-1.0.1.zip b/ec2/third_party/mock-1.0.1.zip new file mode 100644 index 000000000000..8233fad23ca6 Binary files /dev/null and b/ec2/third_party/mock-1.0.1.zip differ diff --git a/ec2/third_party/moto-0.2.11.zip b/ec2/third_party/moto-0.2.11.zip new file mode 100644 index 000000000000..0f3780462662 Binary files /dev/null and b/ec2/third_party/moto-0.2.11.zip differ