yutsuo
/
kafka
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Phase V Task 10

master
yutsuo 7 years ago
parent
a261ed7acd
commit
3786e67e84
2 changed files with 835 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 223
      cheatsheet.sh
  2. 612
      kafka_rev1-1.json

223
cheatsheet.sh
Unescape View File

@ -0,0 +1,223 @@
seq 1000 | sh kafka-console-producer.sh \
--request-required-acks 1 --broker-list kafka:9092 --topic test \
&& echo 'Produced 100 messages.'
sh kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic test --from-beginning
#!/bin/bash
for i in {1..1000}
do
echo "Hello $i times"
done
#!/bin/bash
for i in {1..1000}; do echo "Hello $i times" | sh kafka-console-producer.sh --request-required-acks 1 --broker-list kafka:9092 --topic test; done
for i in {1..10}; do echo "Hello $i times"; done
for i in {1..5}; do history | grep stack; done
sh $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 4 --topic partitioned
for i in {1..1000}; do echo "Hello $i times" | sh kafka-console-producer.sh --request-required-acks 1 --broker-list kafka:9092 --topic partitioned; done
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 0 --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 1 --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 2 --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 3 --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --from-beginning;
sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --group consumer_group_01 --from-beginning;
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 3 --partitions 3 --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --describe --zookeeper zookeeper:2181 --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper zookeeper:2181
$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list 172.17.0.5:9092, 172.17.0.2:9092, 172.17.0.2:9092 --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --from-beginning --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $ZK --replication-factor 3 --partitions 6 --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --describe --zookeeper $ZK --topic MultiBrokerTopic
$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper $ZK
$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list=`broker-list.sh` --topic MultiBrokerTopic
# broker-list.sh
docker port "c15aaffca333" 9092 | sed -e "s/0.0.0.0:/172.17.0.2:/g"
172.17.0.2:32768
docker port "952cb4135fed" 9092 | sed -e "s/0.0.0.0:/172.17.0.5:/g"
172.17.0.5:32770
docker port "83eeda4e8ed9" 9092 | sed -e "s/0.0.0.0:/172.17.0.4:/g"
172.17.0.4:32769
$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list 172.19.0.1:32774, 172.19.0.1:32776, 172.19.0.1:32775 --topic MultiBrokerTopic
docker swarm init --advertise-addr 172.17.0.1 --listen-addr 0.0.0.0
start-kafka-shell.sh <DOCKER_HOST_IP> <ZK_HOST:ZK_PORT>
start-kafka-shell.sh 172.17.0.1 zookeeper:2181
start-kafka-shell.sh 172.0.0.1 zookeeper:2181
$KAFKA_HOME/bin/kafka-topics.sh --create --topic topic --partitions 6 --zookeeper $ZK --replication-factor 3
/sbin/ip route|awk '/default/ { print $3 }'
172.19.0.1:32774, 172.19.0.1:32776, 172.19.0.1:32775
zookeeper-shell.sh zookeeper:2181 <<< "ls /brokers/ids"
docker inspect --format '{{ .NetworkSettings.IPAddress }}' <CONTAINER ID>
docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $INSTANCE_ID
172.17.0.5:9092, 172.17.0.2:9092, 172.17.0.2:9092
172.17.0.1:32770, 172.17.0.1:32768, 172.17.0.1:32769
Error while fetching metadata with correlation LEADER_NOT_AVAILABLE
usr/bin/kafka-topics --create --zookeeper localhost:22181 --replication-factor 3 --partitions 3 --topic MultiBrokerTopic
usr/bin/kafka-topics --describe --zookeeper localhost:22181 --topic MultiBrokerTopic
usr/bin/kafka-topics --delete --zookeeper localhost:22181 --topic MultiBrokerTopic
usr/bin/kafka-topics --list --zookeeper localhost:22181
usr/bin/kafka-console-producer --broker-list localhost:19092, localhost:29092, localhost:39092 --topic MultiBrokerTopic
usr/bin/kafka-console-consumer --bootstrap-server localhost:39092 --from-beginning --group consumer_group_01 --topic MultiBrokerTopic
usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --list
usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --describe --group consumer_group_01
docker exec -ti a1286359925c
docker run confluentinc/cp-kafka:latest
usr/bin/zookeeper-shell localhost:22181 <<< "ls /brokers/ids"
for i in {1..1000}; do echo "Hello $i times" | usr/bin/kafka-console-producer --request-required-acks 1 --broker-list localhost:19092, localhost:29092, localhost:39092 --topic MultiBrokerTopic; done
for i in {1..1000}; do echo "Hello $i times" | usr/bin/kafka-console-producer \
--request-required-acks 1 \
--broker-list localhost:19092, localhost:29092, localhost:39092 \
--topic MultiBrokerTopic; done
$KAFKA_HOME/bin/kafka-console-producer.sh \
--broker-list localhost:9092 \
--topic my-topic \
--property "parse.key=true" \
--property "key.separator=:"
key1:value1
key2:value2
key3:value3
usr/bin/kafka-topics \
--create \
--zookeeper localhost:22181 \
--replication-factor 3 \
--partitions 1 \
--topic KeyedTopic
usr/bin/kafka-topics --describe --zookeeper localhost:22181 --topic KeyedTopic
usr/bin/kafka-topics \
--delete \
--zookeeper localhost:22181 \
--topic KeyedTopic
usr/bin/kafka-topics --list --zookeeper localhost:22181
usr/bin/kafka-console-producer --broker-list localhost:19092, localhost:29092, localhost:39092 --topic KeyedTopic
usr/bin/kafka-console-consumer --bootstrap-server localhost:39092 --from-beginning --group consumer_group_01 --topic KeyedTopic
usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --list
usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --describe --group consumer_group_01
usr/bin/kafka-console-consumer \
--bootstrap-server localhost:39092 \
--from-beginning \
--group consumer_group_01 \
--topic KeyedTopic
for k in {a,b,c}; do \
for i in {1..5}; do \
echo "$k:Hello $i times using key $k" \
| usr/bin/kafka-console-producer \
--request-required-acks 1 \
--broker-list localhost:19092, localhost:29092, localhost:39092 \
--property "parse.key=true" \
--property "key.separator=:" \
--topic KeyedTopic; done; \
done
# for i in a b; do for j in a b; do echo $j; done; done
# java -javaagent:./jmx_prometheus_javaagent-0.11.0.jar=8080:config.yaml -jar yourJar.jar
# KAFKA_OPTS: -javaagent:/usr/app/jmx_prometheus_javaagent.jar=7071:/usr/app/prom-jmx-agent-config.yml
KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.0.jar=7071:/usr/src/config.yml
java -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.0.jar=7071:/usr/src/config.yml -jar yourJar.jar
java -javaagent:./jmx_prometheus_javaagent-0.11.0.jar=7071:config.yml
java -javaagent:./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7071:config.yml
jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
sum without(instance)(rate(kafka_server_brokertopicmetrics_messagesin_total{job="kafka-1",topic!=""}[5m]))
sum without(instance)(rate(kafka_server_brokertopicmetrics_count{job="kafka-1",topic!=""}[5m]))
sum (kafka_server_brokertopicmetrics_count{topic!="",topic!="__consumer_offsets",topic!="__confluent.support.metrics"}) by (topic)
kafka_server_brokertopicmetrics_count{
alias=~"kafka-1|kafka-2|kafka-3",
instance=~"localhost:7071|localhost:7072|localhost:7073",
job=~"kafka-1|kafka-2|kafka-3",
name="MessagesInPerSec",
topic="MultiBrokerTopic"}
# bin/kafka-configs.sh --zookeeper localhost:2181 --entity-type topics --entity-name my-topic --alter --add-config retention.ms=128000
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--add-config cleanup.policy=compact
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--add-config delete.retention.ms=10000
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--add-config min.cleanable.dirty.ratio=0.01
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--add-config 'cleanup.policy=compact, delete.retention.ms=10000, segment.ms=100, segment.bytes=20000, min.cleanable.dirty.ratio=0.01, min.compaction.lag.ms=100, cleaner.min.compaction.lag.ms=100'
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--alter \
--delete-config 'segment.bytes'
/usr/bin/kafka-configs \
--zookeeper localhost:22181 \
--entity-type topics \
--entity-name KeyedTopic \
--describe
#tmux commands
setw synchronize-panes on
setw synchronize-panes off
setw -g mouse on
setw -g mouse off
docker run -ti --network host confluentinc/cp-kafka:latest bash
{__name__=~".+"}
{__name__=~".+",topic="__consumer_offsets"}
docker run --network host -e BURROW_BACKEND=http://localhost:8000 joway/burrow-dashboard:latest
code /etc/systemd/system/docker.service.d/http-proxy.conf

612
kafka_rev1-1.json
Unescape View File

@ -0,0 +1,612 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Kafka resource usage and throughput",
"editable": true,
"gnetId": 721,
"graphTooltip": 0,
"id": 3,
"links": [],
"panels": [
{
"aliasColors": {
"localhost:7071": "#629E51"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 0
},
"id": 1,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(process_cpu_seconds_total{job=~\"kafka-1|kafka-2|kafka-3\"}[1m])",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"metric": "process_cpu_seconds_total",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "CPU Usage",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "Cores",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"localhost:7071": "#BA43A9"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 0
},
"id": 2,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum without(area)(jvm_memory_bytes_used{job=~\"kafka-1|kafka-2|kafka-3\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"metric": "jvm_memory_bytes_used",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "JVM Memory Used",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": "Memory",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"localhost:7071": "#890F02"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 0
},
"id": 3,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=~\"kafka-1|kafka-2|kafka-3\"}[5m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{instance}}",
"metric": "jvm_gc_collection_seconds_sum",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Time spent in GC",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "percentunit",
"label": "% time in GC",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 7
},
"id": 4,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(\nkafka_server_brokertopicmetrics_count{\nname=\"MessagesInPerSec\",\ntopic!=\"__consumer_offsets\",\ntopic!=\"__confluent.support.metrics\",\ntopic!=\"\"\n}\n) by (topic)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{topic}}",
"metric": "kafka_server_brokertopicmetrics_messagesin_total",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Messages In Per Topic",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "Messages/s",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 7
},
"id": 5,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum without(instance)(rate(kafka_server_brokertopicmetrics_bytesin_total{job=\"kafka-1\",topic!=\"\"}[5m]))",
"intervalFactor": 2,
"legendFormat": "{{topic}}",
"metric": "kafka_server_brokertopicmetrics_bytesin_total",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Bytes In Per Topic",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "Bps",
"label": "Bytes/s",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 7
},
"id": 6,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum without(instance)(rate(kafka_server_brokertopicmetrics_bytesout_total{job=\"kafka-1\",topic!=\"\"}[5m]))",
"intervalFactor": 2,
"legendFormat": "{{topic}}",
"metric": "kafka_server_brokertopicmetrics_bytesin_total",
"refId": "A",
"step": 4
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Bytes Out Per Topic",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "Bps",
"label": "Bytes/s",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "browser",
"title": "Kafka Overview",
"uid": "KmnO7Muiz",
"version": 2
}
Write Preview
Loading…
Cancel
Save