ELK 日志搭建配置（docker方式）

演示目标图：

配置

elasticsearch

version: '3.8'

networks:
  common-network:
    external: true

services:
  elasticsearch:
    image: elasticsearch:7.12.1
    container_name: ${CONTAINER_NAME}
    deploy:
      resources:
        limits:
          cpus: ${CPUS:-1}
          memory: ${MEMORY_LIMIT:-512M}
    networks:
      - common-network
    ports:
      - "${PANEL_APP_PORT_HTTP}:9200"
    restart: always
    volumes:
      - ${DOCKER_VOLUMES_HOME}/${IMAGE_NAME}/${PANEL_APP_PORT_HTTP}/plugins:/usr/share/elasticsearch/plugins
      - ${DOCKER_VOLUMES_HOME}/${IMAGE_NAME}/${PANEL_APP_PORT_HTTP}/data:/usr/share/elasticsearch/data
      - ${DOCKER_VOLUMES_HOME}/${IMAGE_NAME}/${PANEL_APP_PORT_HTTP}/logs:/usr/share/elasticsearch/logs
      - ${DOCKER_VOLUMES_HOME}/${IMAGE_NAME}/${PANEL_APP_PORT_HTTP}/config:/usr/share/elasticsearch/config

      - /etc/timezone:/etc/timezone:ro
      - /etc/localtime:/etc/localtime:ro
    environment:
      - TZ=Asia/Shanghai
      - discovery.type=single-node 
      #- xpack.security.enabled=${XPACK_SECURITY_ENABLED}   # ✅ 开启安全认证
      #- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}  # ✅ 设置 elastic 用户密码
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"

kibana

version: '3.8'

networks:
  common-network:
    external: true

services:
  kibana:
    image: kibana:7.12.1
    container_name: ${CONTAINER_NAME}
    deploy:
      resources:
        limits:
          cpus: ${CPUS:-1}
          memory: ${MEMORY_LIMIT:-512M}
    networks:
      - common-network

    environment:
      - ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOSTS}
      - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME}
      - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD}
    ports:
      - "${PANEL_APP_PORT_HTTP}:5601"

logstash

version: '3.8'

networks:
  common-network:
    external: true

services:
  logstash:
    image: logstash:7.12.1
    container_name: ${CONTAINER_NAME}
    deploy:
      resources:
        limits:
          cpus: ${CPUS:-1}
          memory: ${MEMORY_LIMIT:-512M}
    networks:
      - common-network

    volumes:
      - ${DOCKER_VOLUMES_HOME}/${IMAGE_NAME}/${PANEL_APP_PORT_HTTP}/config:/usr/share/logstash/config
      - ${DOCKER_VOLUMES_HOME}/${IMAGE_NAME}/${PANEL_APP_PORT_HTTP}/pipeline:/usr/share/logstash/pipeline
      
    environment:
      - "LS_JAVA_OPTS=${LS_JAVA_OPTS}"

    ports:
      - "${PANEL_APP_PORT_HTTP}:9600"
      - "${PANEL_APP_PORT_HTTP_BEATS}:5044"

logstash 配置
config文件夹下配置
logstash-sample.cofig

input {
  beats {
    port => 5044
  }
}

output {
  elasticsearch {
    hosts => ["http://elasticsearch:9200"]
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

pipeline文件夹下配置
logstash.conf

#input {
#  beats {
#    port => 5044
#  }
#}

#output {
#  stdout {
#    codec => rubydebug
#  }
#}

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 5044
    codec => json_lines
  }
}

output {
  elasticsearch {
    hosts => ["http://elasticsearch:9200"]
    #index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
  index => "logstash"
    #user => "elastic"
    #password => "changeme"
  }
}

JAVA服务配置
yml 配置

logstash:
  addr: 127.0.0.1:5044
  file: var/log/${spring.application.name}/${spring.profiles.active}/${spring.application.name}.log
  fileNamePattern: var/log/${spring.application.name}/${spring.profiles.active}/${spring.application.name}-logstash.%d{yyyy-MM-dd}.log
  maxHistory: 30
  totalSizeCap: 10MB

logback-spring.xml 配置

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE configuration>
<configuration>
    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
    <include resource="org/springframework/boot/logging/logback/base.xml"/>

    <springProperty scope="context" name="appName" source="spring.application.name"/>
    <springProperty scope="context" name="logPath" source="logging.file.name"/>
    <springProperty scope="context" name="dev" source="spring.profiles.active"/>
    <springProperty scope="context" name="logstashAddr" source="logstash.addr"/>
    <springProperty scope="context" name="logstashFile" source="logstash.file"/>
    <springProperty scope="context" name="logstashMaxHistory" source="logstash.maxHistory"/>
    <springProperty scope="context" name="logstashTotalSizeCap" source="logstash.totalSizeCap"/>


    <!-- 日志在工程中的输出位置 -->
    <property name="LOG_FILE" value="${BUILD_FOLDER:-build}/${appName}"/>

    <!-- 控制台的日志输出样式 -->
    <property name="CONSOLE_LOG_PATTERN"
              value="%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>

    <!-- 控制台输出 -->
    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>INFO</level>
        </filter>
        <!-- 日志输出编码 -->
        <encoder>
            <!--输出格式化-->
            <pattern>[%X{TRACE_ID:-DEFAULT}]  %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
            <charset>utf8</charset>
        </encoder>
    </appender>

    <!-- 根据日志文件地址是否为有效地址，决定是否启用 RollingFileAppender -->
    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${logstashFile:-/var/log/${appName}/${dev}/${appName}-logstash.log}</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!-- 设置日志文件名 -->
            <fileNamePattern>${logstashFileNamePattern:-/var/log/${appName}/${dev}/${appName}-logstash.%d{yyyy-MM-dd}.log}</fileNamePattern>
            <!-- 最多保留 30 天的日志 -->
            <maxHistory>${logstashMaxHistory:-30}</maxHistory>
            <!--设置日志文件大小，超过就重新生成文件，默认10M-->
            <totalSizeCap>${logstashTotalSizeCap:-10MB}</totalSizeCap>
        </rollingPolicy>
        <encoder>
            <pattern>[%X{TRACE_ID:-DEFAULT}]  %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
        </encoder>
    </appender>


    <!--logstash配置-->
    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>${logstashAddr:-127.0.0.1:4560}</destination>
        <!-- 日志输出编码 -->
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>Asia/Shanghai</timeZone>
                </timestamp>
                <pattern>
                    <pattern>
                        {
                        "dev": "${dev:-}",
                        "logLevel": "%level",
                        "serviceName": "${appName:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger{40}",
                        "rest": "%message"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
        <!--<encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder"/>-->
        <!--当有多个LogStash服务时，设置访问策略为轮询-->
        <connectionStrategy>
            <roundRobin>
                <connectionTTL>5 minutes</connectionTTL>
            </roundRobin>
        </connectionStrategy>
    </appender>

    <root level="INFO">
        <appender-ref ref="LOGSTASH"/>
        <appender-ref ref="CONSOLE"/>
    </root>

    <!--禁掉一些包-->
    <logger name="com.alibaba.nacos" level="OFF"></logger>
    <springProfile name="test">
        <root level="INFO">
            <appender-ref ref="CONSOLE"/>
            <appender-ref ref="FILE"/>
        </root>
    </springProfile>

</configuration>