yutsuo
/
kafka
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3 Commits
1 Branch
0 Tags
876 KiB
 
 
Branch: master
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'master'
${ noResults }
kafka/cheatsheet.sh

230 lines
9.0 KiB
Raw Permalink Blame History

seq 1000 | sh kafka-console-producer.sh \
--request-required-acks 1 --broker-list kafka:9092 --topic test \
&& echo 'Produced 100 messages.'
sh kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic test --from-beginning
#!/bin/bash
for i in {1..1000}
do
echo "Hello $i times"
done
#!/bin/bash
for i in {1..1000}; do echo "Hello $i times" | sh kafka-console-producer.sh --request-required-acks 1 --broker-list kafka:9092 --topic test; done
for i in {1..10}; do echo "Hello $i times"; done
for i in {1..5}; do history | grep stack; done
sh $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 4 --topic partitioned
for i in {1..1000}; do echo "Hello $i times" | sh kafka-console-producer.sh --request-required-acks 1 --broker-list kafka:9092 --topic partitioned; done
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 0 --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 1 --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 2 --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 3 --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --group consumer_group_01 --from-beginning;
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 3 --partitions 3 --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --describe --zookeeper zookeeper:2181 --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper zookeeper:2181
$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list 172.17.0.5:9092, 172.17.0.2:9092, 172.17.0.2:9092 --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --from-beginning --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $ZK --replication-factor 3 --partitions 6 --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --describe --zookeeper $ZK --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper $ZK
$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list=`broker-list.sh` --topic MultiBrokerTopic
# broker-list.sh
docker port "c15aaffca333" 9092 | sed -e "s/0.0.0.0:/172.17.0.2:/g"
172.17.0.2:32768
docker port "952cb4135fed" 9092 | sed -e "s/0.0.0.0:/172.17.0.5:/g"
172.17.0.5:32770
docker port "83eeda4e8ed9" 9092 | sed -e "s/0.0.0.0:/172.17.0.4:/g"
172.17.0.4:32769
$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list 172.19.0.1:32774, 172.19.0.1:32776, 172.19.0.1:32775 --topic MultiBrokerTopic
docker swarm init --advertise-addr 172.17.0.1 --listen-addr 0.0.0.0
start-kafka-shell.sh <DOCKER_HOST_IP> <ZK_HOST:ZK_PORT>
start-kafka-shell.sh 172.17.0.1 zookeeper:2181
start-kafka-shell.sh 172.0.0.1 zookeeper:2181
$KAFKA_HOME/bin/kafka-topics.sh --create --topic topic --partitions 6 --zookeeper $ZK --replication-factor 3
/sbin/ip route|awk '/default/ { print $3 }'
172.19.0.1:32774, 172.19.0.1:32776, 172.19.0.1:32775
zookeeper-shell.sh zookeeper:2181 <<< "ls /brokers/ids"
docker inspect --format '{{ .NetworkSettings.IPAddress }}' <CONTAINER ID>
docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $INSTANCE_ID
172.17.0.5:9092, 172.17.0.2:9092, 172.17.0.2:9092
172.17.0.1:32770, 172.17.0.1:32768, 172.17.0.1:32769
Error while fetching metadata with correlation LEADER_NOT_AVAILABLE
usr/bin/kafka-topics --create --zookeeper localhost:22181 --replication-factor 3 --partitions 3 --topic MultiBrokerTopic
usr/bin/kafka-topics --describe --zookeeper localhost:22181 --topic MultiBrokerTopic
usr/bin/kafka-topics --delete --zookeeper localhost:22181 --topic MultiBrokerTopic
usr/bin/kafka-topics --list --zookeeper localhost:22181
usr/bin/kafka-console-producer --broker-list localhost:19092, localhost:29092, localhost:39092 --topic MultiBrokerTopic
usr/bin/kafka-console-consumer --bootstrap-server localhost:39092 --from-beginning --group consumer_group_01 --topic MultiBrokerTopic
usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --list
usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --describe --group consumer_group_01
docker exec -ti a1286359925c
docker run confluentinc/cp-kafka:latest
usr/bin/zookeeper-shell localhost:22181 <<< "ls /brokers/ids"
for i in {1..1000}; do echo "Hello $i times" | usr/bin/kafka-console-producer --request-required-acks 1 --broker-list localhost:19092, localhost:29092, localhost:39092 --topic MultiBrokerTopic; done
for i in {1..1000}; do echo "Hello $i times" | usr/bin/kafka-console-producer \
--request-required-acks 1 \
--broker-list localhost:19092, localhost:29092, localhost:39092 \
--topic MultiBrokerTopic; done
# $KAFKA_HOME/bin/kafka-console-producer.sh \
# --broker-list localhost:9092 \
# --topic my-topic \
# --property "parse.key=true" \
# --property "key.separator=:"
# key1:value1
# key2:value2
# key3:value3
usr/bin/kafka-topics \
--create \
--zookeeper localhost:22181 \
--replication-factor 3 \
--partitions 1 \
--topic KeyedTopic
usr/bin/kafka-topics --describe --zookeeper localhost:22181 --topic KeyedTopic
usr/bin/kafka-topics \
--delete \
--zookeeper localhost:22181 \
--topic KeyedTopic
usr/bin/kafka-topics --list --zookeeper localhost:22181
usr/bin/kafka-console-producer --broker-list localhost:19092, localhost:29092, localhost:39092 --topic KeyedTopic
usr/bin/kafka-console-consumer \
--bootstrap-server localhost:39092 \
--from-beginning \
--group consumer_group_02 \
--topic KeyedTopic
usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --list
usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --describe --group consumer_group_01
usr/bin/kafka-console-consumer \
--bootstrap-server localhost:39092 \
--from-beginning \
--group consumer_group_01 \
--topic KeyedTopic
\
for k in {a,b,c}; do \
for i in {1..100}; do \
echo "$k:Hello $i times using key $k" \
| usr/bin/kafka-console-producer \
--request-required-acks 1 \
--broker-list localhost:19092, localhost:29092, localhost:39092 \
--property "parse.key=true" \
--property "key.separator=:" \
--topic KeyedTopic; done; \
done
# for i in a b; do for j in a b; do echo $j; done; done
# java -javaagent:./jmx_prometheus_javaagent-0.11.0.jar=8080:config.yaml -jar yourJar.jar
# KAFKA_OPTS: -javaagent:/usr/app/jmx_prometheus_javaagent.jar=7071:/usr/app/prom-jmx-agent-config.yml
KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.0.jar=7071:/usr/src/config.yml
java -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.0.jar=7071:/usr/src/config.yml -jar yourJar.jar
java -javaagent:./jmx_prometheus_javaagent-0.11.0.jar=7071:config.yml
java -javaagent:./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7071:config.yml
jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
sum without(instance)(rate(kafka_server_brokertopicmetrics_messagesin_total{job="kafka-1",topic!=""}[5m]))
sum without(instance)(rate(kafka_server_brokertopicmetrics_count{job="kafka-1",topic!=""}[5m]))
sum (kafka_server_brokertopicmetrics_count{topic!="",topic!="__consumer_offsets",topic!="__confluent.support.metrics"}) by (topic)
kafka_server_brokertopicmetrics_count{
alias=~"kafka-1|kafka-2|kafka-3",
instance=~"localhost:7071|localhost:7072|localhost:7073",
job=~"kafka-1|kafka-2|kafka-3",
name="MessagesInPerSec",
topic="MultiBrokerTopic"}
# bin/kafka-configs.sh --zookeeper localhost:2181 --entity-type topics --entity-name my-topic --alter --add-config retention.ms=128000
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--add-config cleanup.policy=compact
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--add-config delete.retention.ms=10000
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--add-config min.cleanable.dirty.ratio=0.01
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--add-config 'cleanup.policy=compact, delete.retention.ms=10000, segment.ms=100, min.cleanable.dirty.ratio=0.01, min.compaction.lag.ms=100, min.compaction.lag.ms=100'
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--delete-config 'segment.bytes'
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--describe
#tmux commands
setw synchronize-panes on
setw synchronize-panes off
setw -g mouse on
setw -g mouse off
docker run -ti --network host confluentinc/cp-kafka:latest bash
{__name__=~".+"}
{__name__=~".+",topic="__consumer_offsets"}
docker run --network host -e BURROW_BACKEND=http://localhost:8000 joway/burrow-dashboard:latest
code /etc/systemd/system/docker.service.d/http-proxy.conf