mirror of
https://github.com/didi/KnowStreaming.git
synced 2026-01-02 02:02:13 +08:00
Add km module kafka
This commit is contained in:
leewei
25
vagrant/aws/aws-access-keys-commands
Normal file
25
vagrant/aws/aws-access-keys-commands
Normal file
@@ -0,0 +1,25 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
export AWS_IAM_ROLE=$(curl -s http://169.254.169.254/latest/meta-data/iam/info | grep InstanceProfileArn | cut -d '"' -f 4 | cut -d '/' -f 2)
|
||||
export AWS_ACCESS_KEY=$(curl -s http://169.254.169.254/latest/meta-data/iam/security-credentials/$AWS_IAM_ROLE | grep AccessKeyId | awk -F\" '{ print $4 }')
|
||||
export AWS_SECRET_KEY=$(curl -s http://169.254.169.254/latest/meta-data/iam/security-credentials/$AWS_IAM_ROLE | grep SecretAccessKey | awk -F\" '{ print $4 }')
|
||||
export AWS_SESSION_TOKEN=$(curl -s http://169.254.169.254/latest/meta-data/iam/security-credentials/$AWS_IAM_ROLE | grep Token | awk -F\" '{ print $4 }')
|
||||
|
||||
if [ -z "$AWS_ACCESS_KEY" ]; then
|
||||
echo "Failed to populate environment variables AWS_ACCESS_KEY, AWS_SECRET_KEY, and AWS_SESSION_TOKEN."
|
||||
echo "AWS_IAM is currently $AWS_IAM. Double-check that this is correct. If not set, add this command to your .bashrc file:"
|
||||
echo "export AWS_IAM=<my_aws_iam> # put this into your ~/.bashrc"
|
||||
fi
|
||||
29
vagrant/aws/aws-example-Vagrantfile.local
Normal file
29
vagrant/aws/aws-example-Vagrantfile.local
Normal file
@@ -0,0 +1,29 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Use this template Vagrantfile.local for running system tests on aws
|
||||
# To use it, move it to the base kafka directory and rename
|
||||
# it to Vagrantfile.local, and adjust variables as needed.
|
||||
ec2_instance_type = "m3.xlarge"
|
||||
ec2_spot_max_price = "0.266" # On-demand price for instance type
|
||||
enable_hostmanager = false
|
||||
num_zookeepers = 0
|
||||
num_brokers = 0
|
||||
num_workers = 9
|
||||
ec2_keypair_name = kafkatest
|
||||
ec2_keypair_file = ../kafkatest.pem
|
||||
ec2_security_groups = ['kafkatest']
|
||||
ec2_region = 'us-west-2'
|
||||
ec2_ami = "ami-29ebb519"
|
||||
81
vagrant/aws/aws-init.sh
Executable file
81
vagrant/aws/aws-init.sh
Executable file
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script can be used to set up a driver machine on aws from which you will run tests
|
||||
# or bring up your mini Kafka cluster.
|
||||
|
||||
# Install dependencies
|
||||
sudo apt-get install -y \
|
||||
maven \
|
||||
openjdk-8-jdk-headless \
|
||||
build-essential \
|
||||
ruby-dev \
|
||||
zlib1g-dev \
|
||||
realpath \
|
||||
python-setuptools \
|
||||
iperf \
|
||||
traceroute
|
||||
|
||||
base_dir=`dirname $0`/../..
|
||||
|
||||
if [ -z `which vagrant` ]; then
|
||||
echo "Installing vagrant..."
|
||||
wget https://releases.hashicorp.com/vagrant/2.1.5/vagrant_2.1.5_x86_64.deb
|
||||
sudo dpkg -i vagrant_2.1.5_x86_64.deb
|
||||
rm -f vagrant_2.1.5_x86_64.deb
|
||||
fi
|
||||
|
||||
# Install necessary vagrant plugins
|
||||
# Note: Do NOT install vagrant-cachier since it doesn't work on AWS and only
|
||||
# adds log noise
|
||||
|
||||
# Custom vagrant-aws with spot instance support. See https://github.com/mitchellh/vagrant-aws/issues/32
|
||||
wget -nv https://s3-us-west-2.amazonaws.com/confluent-packaging-tools/vagrant-aws-0.7.2.spot.gem -P /tmp
|
||||
vagrant_plugins="/tmp/vagrant-aws-0.7.2.spot.gem vagrant-hostmanager"
|
||||
existing=`vagrant plugin list`
|
||||
for plugin in $vagrant_plugins; do
|
||||
echo $existing | grep $plugin > /dev/null
|
||||
if [ $? != 0 ]; then
|
||||
vagrant plugin install $plugin
|
||||
fi
|
||||
done
|
||||
|
||||
# Create Vagrantfile.local as a convenience
|
||||
if [ ! -e "$base_dir/Vagrantfile.local" ]; then
|
||||
cp $base_dir/vagrant/aws/aws-example-Vagrantfile.local $base_dir/Vagrantfile.local
|
||||
fi
|
||||
|
||||
gradle="gradle-2.2.1"
|
||||
if [ -z `which gradle` ] && [ ! -d $base_dir/$gradle ]; then
|
||||
if [ ! -e $gradle-bin.zip ]; then
|
||||
wget https://services.gradle.org/distributions/$gradle-bin.zip
|
||||
fi
|
||||
unzip $gradle-bin.zip
|
||||
rm -rf $gradle-bin.zip
|
||||
mv $gradle $base_dir/$gradle
|
||||
fi
|
||||
|
||||
# Ensure aws access keys are in the environment when we use a EC2 driver machine
|
||||
LOCAL_HOSTNAME=$(hostname -d)
|
||||
if [[ ${LOCAL_HOSTNAME} =~ .*\.compute\.internal ]]; then
|
||||
grep "AWS ACCESS KEYS" ~/.bashrc > /dev/null
|
||||
if [ $? != 0 ]; then
|
||||
echo "# --- AWS ACCESS KEYS ---" >> ~/.bashrc
|
||||
echo ". `realpath $base_dir/aws/aws-access-keys-commands`" >> ~/.bashrc
|
||||
echo "# -----------------------" >> ~/.bashrc
|
||||
source ~/.bashrc
|
||||
fi
|
||||
fi
|
||||
Reference in New Issue
Block a user