diff --git a/cheatsheet.sh b/cheatsheet.sh new file mode 100644 index 0000000..4b941fb --- /dev/null +++ b/cheatsheet.sh @@ -0,0 +1,223 @@ +seq 1000 | sh kafka-console-producer.sh \ +--request-required-acks 1 --broker-list kafka:9092 --topic test \ +&& echo 'Produced 100 messages.' + +sh kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic test --from-beginning + +#!/bin/bash +for i in {1..1000} +do + echo "Hello $i times" +done + +#!/bin/bash +for i in {1..1000}; do echo "Hello $i times" | sh kafka-console-producer.sh --request-required-acks 1 --broker-list kafka:9092 --topic test; done + +for i in {1..10}; do echo "Hello $i times"; done +for i in {1..5}; do history | grep stack; done + +sh $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 4 --topic partitioned +for i in {1..1000}; do echo "Hello $i times" | sh kafka-console-producer.sh --request-required-acks 1 --broker-list kafka:9092 --topic partitioned; done +sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 0 --from-beginning; +sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 1 --from-beginning; +sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 2 --from-beginning; +sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --partition 3 --from-beginning; + +sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --from-beginning; + + + +sh $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic partitioned --group consumer_group_01 --from-beginning; + +$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 3 --partitions 3 --topic MultiBrokerTopic +$KAFKA_HOME/bin/kafka-topics.sh --describe --zookeeper zookeeper:2181 --topic MultiBrokerTopic +$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper zookeeper:2181 +$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list 172.17.0.5:9092, 172.17.0.2:9092, 172.17.0.2:9092 --topic MultiBrokerTopic +$KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --from-beginning --topic MultiBrokerTopic + +$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $ZK --replication-factor 3 --partitions 6 --topic MultiBrokerTopic +$KAFKA_HOME/bin/kafka-topics.sh --describe --zookeeper $ZK --topic MultiBrokerTopic +$KAFKA_HOME/bin/kafka-topics.sh --list --zookeeper $ZK +$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list=`broker-list.sh` --topic MultiBrokerTopic + +# broker-list.sh +docker port "c15aaffca333" 9092 | sed -e "s/0.0.0.0:/172.17.0.2:/g" +172.17.0.2:32768 + +docker port "952cb4135fed" 9092 | sed -e "s/0.0.0.0:/172.17.0.5:/g" +172.17.0.5:32770 + +docker port "83eeda4e8ed9" 9092 | sed -e "s/0.0.0.0:/172.17.0.4:/g" +172.17.0.4:32769 + +$KAFKA_HOME/bin/kafka-console-producer.sh --broker-list 172.19.0.1:32774, 172.19.0.1:32776, 172.19.0.1:32775 --topic MultiBrokerTopic + +docker swarm init --advertise-addr 172.17.0.1 --listen-addr 0.0.0.0 + +start-kafka-shell.sh +start-kafka-shell.sh 172.17.0.1 zookeeper:2181 +start-kafka-shell.sh 172.0.0.1 zookeeper:2181 + +$KAFKA_HOME/bin/kafka-topics.sh --create --topic topic --partitions 6 --zookeeper $ZK --replication-factor 3 + +/sbin/ip route|awk '/default/ { print $3 }' + +172.19.0.1:32774, 172.19.0.1:32776, 172.19.0.1:32775 + +zookeeper-shell.sh zookeeper:2181 <<< "ls /brokers/ids" + +docker inspect --format '{{ .NetworkSettings.IPAddress }}' + +docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $INSTANCE_ID + +172.17.0.5:9092, 172.17.0.2:9092, 172.17.0.2:9092 +172.17.0.1:32770, 172.17.0.1:32768, 172.17.0.1:32769 + +Error while fetching metadata with correlation LEADER_NOT_AVAILABLE + +usr/bin/kafka-topics --create --zookeeper localhost:22181 --replication-factor 3 --partitions 3 --topic MultiBrokerTopic +usr/bin/kafka-topics --describe --zookeeper localhost:22181 --topic MultiBrokerTopic +usr/bin/kafka-topics --delete --zookeeper localhost:22181 --topic MultiBrokerTopic +usr/bin/kafka-topics --list --zookeeper localhost:22181 +usr/bin/kafka-console-producer --broker-list localhost:19092, localhost:29092, localhost:39092 --topic MultiBrokerTopic +usr/bin/kafka-console-consumer --bootstrap-server localhost:39092 --from-beginning --group consumer_group_01 --topic MultiBrokerTopic +usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --list +usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --describe --group consumer_group_01 + +docker exec -ti a1286359925c +docker run confluentinc/cp-kafka:latest + +usr/bin/zookeeper-shell localhost:22181 <<< "ls /brokers/ids" + +for i in {1..1000}; do echo "Hello $i times" | usr/bin/kafka-console-producer --request-required-acks 1 --broker-list localhost:19092, localhost:29092, localhost:39092 --topic MultiBrokerTopic; done + +for i in {1..1000}; do echo "Hello $i times" | usr/bin/kafka-console-producer \ +--request-required-acks 1 \ +--broker-list localhost:19092, localhost:29092, localhost:39092 \ +--topic MultiBrokerTopic; done + +$KAFKA_HOME/bin/kafka-console-producer.sh \ + --broker-list localhost:9092 \ + --topic my-topic \ + --property "parse.key=true" \ + --property "key.separator=:" +key1:value1 +key2:value2 +key3:value3 + +usr/bin/kafka-topics \ +--create \ +--zookeeper localhost:22181 \ +--replication-factor 3 \ +--partitions 1 \ +--topic KeyedTopic + +usr/bin/kafka-topics --describe --zookeeper localhost:22181 --topic KeyedTopic + +usr/bin/kafka-topics \ +--delete \ +--zookeeper localhost:22181 \ +--topic KeyedTopic + +usr/bin/kafka-topics --list --zookeeper localhost:22181 +usr/bin/kafka-console-producer --broker-list localhost:19092, localhost:29092, localhost:39092 --topic KeyedTopic +usr/bin/kafka-console-consumer --bootstrap-server localhost:39092 --from-beginning --group consumer_group_01 --topic KeyedTopic +usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --list +usr/bin/kafka-consumer-groups --bootstrap-server localhost:39092 --describe --group consumer_group_01 + +usr/bin/kafka-console-consumer \ +--bootstrap-server localhost:39092 \ +--from-beginning \ +--group consumer_group_01 \ +--topic KeyedTopic + + +for k in {a,b,c}; do \ + for i in {1..5}; do \ + echo "$k:Hello $i times using key $k" \ + | usr/bin/kafka-console-producer \ + --request-required-acks 1 \ + --broker-list localhost:19092, localhost:29092, localhost:39092 \ + --property "parse.key=true" \ + --property "key.separator=:" \ + --topic KeyedTopic; done; \ +done + +# for i in a b; do for j in a b; do echo $j; done; done + +# java -javaagent:./jmx_prometheus_javaagent-0.11.0.jar=8080:config.yaml -jar yourJar.jar +# KAFKA_OPTS: -javaagent:/usr/app/jmx_prometheus_javaagent.jar=7071:/usr/app/prom-jmx-agent-config.yml +KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.0.jar=7071:/usr/src/config.yml +java -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.0.jar=7071:/usr/src/config.yml -jar yourJar.jar +java -javaagent:./jmx_prometheus_javaagent-0.11.0.jar=7071:config.yml +java -javaagent:./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7071:config.yml +jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar + +sum without(instance)(rate(kafka_server_brokertopicmetrics_messagesin_total{job="kafka-1",topic!=""}[5m])) +sum without(instance)(rate(kafka_server_brokertopicmetrics_count{job="kafka-1",topic!=""}[5m])) +sum (kafka_server_brokertopicmetrics_count{topic!="",topic!="__consumer_offsets",topic!="__confluent.support.metrics"}) by (topic) + +kafka_server_brokertopicmetrics_count{ +alias=~"kafka-1|kafka-2|kafka-3", +instance=~"localhost:7071|localhost:7072|localhost:7073", +job=~"kafka-1|kafka-2|kafka-3", +name="MessagesInPerSec", +topic="MultiBrokerTopic"} + +# bin/kafka-configs.sh --zookeeper localhost:2181 --entity-type topics --entity-name my-topic --alter --add-config retention.ms=128000 + +/usr/bin/kafka-configs \ +--zookeeper localhost:22181 \ +--entity-type topics \ +--entity-name KeyedTopic \ +--alter \ +--add-config cleanup.policy=compact + +/usr/bin/kafka-configs \ +--zookeeper localhost:22181 \ +--entity-type topics \ +--entity-name KeyedTopic \ +--alter \ +--add-config delete.retention.ms=10000 + +/usr/bin/kafka-configs \ +--zookeeper localhost:22181 \ +--entity-type topics \ +--entity-name KeyedTopic \ +--alter \ +--add-config min.cleanable.dirty.ratio=0.01 + +/usr/bin/kafka-configs \ +--zookeeper localhost:22181 \ +--entity-type topics \ +--entity-name KeyedTopic \ +--alter \ +--add-config 'cleanup.policy=compact, delete.retention.ms=10000, segment.ms=100, segment.bytes=20000, min.cleanable.dirty.ratio=0.01, min.compaction.lag.ms=100, cleaner.min.compaction.lag.ms=100' + +/usr/bin/kafka-configs \ +--zookeeper localhost:22181 \ +--entity-type topics \ +--entity-name KeyedTopic \ +--alter \ +--delete-config 'segment.bytes' + +/usr/bin/kafka-configs \ +--zookeeper localhost:22181 \ +--entity-type topics \ +--entity-name KeyedTopic \ +--describe + +#tmux commands +setw synchronize-panes on +setw synchronize-panes off +setw -g mouse on +setw -g mouse off + +docker run -ti --network host confluentinc/cp-kafka:latest bash + +{__name__=~".+"} +{__name__=~".+",topic="__consumer_offsets"} + +docker run --network host -e BURROW_BACKEND=http://localhost:8000 joway/burrow-dashboard:latest + +code /etc/systemd/system/docker.service.d/http-proxy.conf \ No newline at end of file diff --git a/kafka_rev1-1.json b/kafka_rev1-1.json new file mode 100644 index 0000000..6530183 --- /dev/null +++ b/kafka_rev1-1.json @@ -0,0 +1,612 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Kafka resource usage and throughput", + "editable": true, + "gnetId": 721, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "aliasColors": { + "localhost:7071": "#629E51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=~\"kafka-1|kafka-2|kafka-3\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "process_cpu_seconds_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "localhost:7071": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(area)(jvm_memory_bytes_used{job=~\"kafka-1|kafka-2|kafka-3\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "jvm_memory_bytes_used", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "JVM Memory Used", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Memory", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "localhost:7071": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(gc)(rate(jvm_gc_collection_seconds_sum{job=~\"kafka-1|kafka-2|kafka-3\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "jvm_gc_collection_seconds_sum", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Time spent in GC", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "% time in GC", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\nkafka_server_brokertopicmetrics_count{\nname=\"MessagesInPerSec\",\ntopic!=\"__consumer_offsets\",\ntopic!=\"__confluent.support.metrics\",\ntopic!=\"\"\n}\n) by (topic)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{topic}}", + "metric": "kafka_server_brokertopicmetrics_messagesin_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Messages In Per Topic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Messages/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance)(rate(kafka_server_brokertopicmetrics_bytesin_total{job=\"kafka-1\",topic!=\"\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "{{topic}}", + "metric": "kafka_server_brokertopicmetrics_bytesin_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes In Per Topic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Bytes/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 6, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance)(rate(kafka_server_brokertopicmetrics_bytesout_total{job=\"kafka-1\",topic!=\"\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "{{topic}}", + "metric": "kafka_server_brokertopicmetrics_bytesin_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Out Per Topic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Bytes/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kafka Overview", + "uid": "KmnO7Muiz", + "version": 2 +} \ No newline at end of file