日志管理 #

应用/服务
    ↓
rsylog/journald
    ↓
日志文件
    ↓
日志分析工具

/var/log/syslog       # 系统日志
/var/log/auth.log     # 认证日志
/var/log/kern.log    # 内核日志
/var/log/dmesg       # 启动日志
/var/log/messages    # 消息日志
/var/log/daemon.log  # 守护进程日志
/var/log/nginx/      # Nginx 日志
/var/log/apache2/    # Apache 日志

# 查看 rsyslog 状态
systemctl status rsyslog

# 配置文件
cat /etc/rsyslog.conf

# 模块配置
$ModLoad imuxsock
$ModLoad imklog
$ModLoad imudp
$UDPServerRun 514

# 规则配置
*.info;mail.none;authpriv.none /var/log/messages
authpriv.* /var/log/secure
mail.* /var/log/maillog
cron.* /var/log/cron

# 格式
# facility.priority action

# facility（设施）
auth        # 认证
authpriv    # 认证（私有）
cron        # 定时任务
daemon      # 守护进程
kern        # 内核
lpr         # 打印
mail        # 邮件
user        # 用户
local0-7   # 本地设施

# priority（优先级）
debug       # 调试
info        # 信息
notice      # 注意
warn        # 警告
err         # 错误
crit        # 严重
alert       # 警报
emerg       # 紧急

# action（动作）
/var/log/file       # 写入文件
@remotehost         # 发送到远程
*                   # 所有用户

# 客户端配置
sudo vim /etc/rsyslog.conf

# 发送规则
*.* @@remote-server:514

# 重启服务
sudo systemctl restart rsyslog

# 服务端配置
sudo vim /etc/rsyslog.conf

# 启用模块
$ModLoad imtcp
$InputTCPServerRun 514

# 接收规则
$template RemoteLogs,"/var/log/%HOSTNAME%/%PROGRAMNAME%.log"
*.* ?RemoteLogs

# 重启服务
sudo systemctl restart rsyslog

# 查看 journald 配置
cat /etc/systemd/journald.conf

# 常用配置项
[Journal]
SystemMaxUse=500M
RuntimeMaxUse=200M
SystemMaxFileSize=50M
MaxRetentionSec=30day
ForwardToSyslog=yes

# 查看日志
journalctl

# 查看启动日志
journalctl -b

# 查看上次启动
journalctl -b -1

# 查看指定服务
journalctl -u nginx
journalctl -u nginx -u mysql

# 按时间过滤
journalctl --since "2024-01-01"
journalctl --until "2024-01-02"
journalctl --since "1 hour ago"

# 按优先级
journalctl -p err
journalctl -p warning..emerg

# 实时跟踪
journalctl -f

# 清理日志
journalctl --vacuum-size=100M
journalctl --vacuum-time=7d

# 查看磁盘使用
journalctl --disk-usage

# 设置大小限制
sudo vim /etc/systemd/journald.conf
[Journal]
SystemMaxUse=1G
RuntimeMaxUse=500M

# 安装
sudo apt install logrotate

# 查看配置
cat /etc/logrotate.conf

# 查看子配置
ls /etc/logrotate.d/

# 基本配置示例
/var/log/nginx/*.log {
    daily              # 每天轮转
    missingok         # 忽略错误
    rotate 14         # 保留 14 个文件
    compress          # 压缩旧文件
    delaycompress     # 延迟压缩
    notifempty       # 不轮转空文件
    create 0640 www-data adm  # 创建新文件权限
    sharedscripts    # 共享脚本
    postrotate
        [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
    endscript
}

# 创建自定义配置
sudo vim /etc/logrotate.d/myapp

/var/log/myapp/*.log {
    daily
    rotate 30
    compress
    delaycompress
    missingok
    notifempty
    create 0640 myapp myapp
    sharedscripts
    postrotate
        systemctl reload myapp > /dev/null 2>&1 || true
    endscript
}

# 测试配置
sudo logrotate -d /etc/logrotate.d/myapp

# 强制执行
sudo logrotate -f /etc/logrotate.d/myapp

# 统计日志行数
wc -l /var/log/syslog

# 统计错误数量
grep -c "error" /var/log/syslog
grep -c "ERROR" /var/log/nginx/error.log

# 统计访问量
wc -l /var/log/nginx/access.log

# 统计独立 IP
awk '{print $1}' /var/log/nginx/access.log | sort | uniq | wc -l

# 登录失败分析
sudo grep "Failed password" /var/log/auth.log | awk '{print $11}' | sort | uniq -c | sort -nr | head -10

# SSH 攻击分析
sudo grep "Invalid user" /var/log/auth.log | awk '{print $10}' | sort | uniq -c | sort -nr | head -10

# Nginx 访问分析
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -nr | head -10

# 磁盘空间分析
du -sh /var/log/* | sort -hr | head -10

# 日志大小分析
ls -lhS /var/log/*.log | head -10

# 安装 goaccess
sudo apt install goaccess

# 实时分析 Nginx 日志
goaccess /var/log/nginx/access.log

# 生成 HTML 报告
goaccess /var/log/nginx/access.log -o /var/www/html/report.html --log-format=COMBINED

# 安装 logwatch
sudo apt install logwatch

# 生成报告
sudo logwatch --output stdout --detail high --range Today --service nginx

# 按关键词搜索
grep "error" /var/log/syslog

# 按正则搜索
grep -E "error|warning|critical" /var/log/syslog

# 多文件搜索
grep -r "error" /var/log/nginx/

# 显示上下文
grep -C 3 "error" /var/log/syslog

# 忽略大小写
grep -i "error" /var/log/syslog

# 配置文件
/var/log/nginx/access.log    # 访问日志
/var/log/nginx/error.log     # 错误日志

# 日志格式配置
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                '$status $body_bytes_sent "$http_referer" '
                '"$http_user_agent" "$http_x_forwarded_for"';

# 条件日志
map $request_uri $loggable {
    ~healthcheck 0;
    default 1;
}

access_log /var/log/nginx/access.log main if=$loggable;

# 查看日志配置
SHOW VARIABLES LIKE 'log%';

# 错误日志
# /var/log/mysql/error.log

# 查询日志（生产环境关闭）
SET GLOBAL general_log = 'ON';
SET GLOBAL general_log_file = '/var/log/mysql/query.log';

# 慢查询日志
SET GLOBAL slow_query_log = 'ON';
SET GLOBAL slow_query_log_file = '/var/log/mysql/slow.log';
SET GLOBAL long_query_time = 2;

# 1. 使用结构化日志
# JSON 格式
{"timestamp":"2024-01-01T10:00:00Z","level":"error","message":"Connection failed","host":"server1"}

# 2. 日志级别
DEBUG   # 调试信息
INFO    # 一般信息
WARNING # 警告
ERROR   # 错误
FATAL   # 致命错误

# 3. 日志轮转配置
# Python logging
import logging
from logging.handlers import RotatingFileHandler

handler = RotatingFileHandler('/var/log/myapp.log', maxBytes=10485760, backupCount=10)
logging.getLogger().addHandler(handler)

# 4. 集中式日志
# ELK Stack
# Filebeat → Elasticsearch ← Kibana

# 限制日志权限
sudo chmod 640 /var/log/auth.log
sudo chmod 640 /var/log/secure

# 保护日志不被删除
sudo chattr +a /var/log/auth.log
sudo chattr +i /var/log/auth.log

# 只读日志
sudo mount -o remount,ro /var/log

# 检查日志篡改
sudo aide --check

# 备份脚本
#!/bin/bash
DATE=$(date +%Y%m%d)
BACKUP_DIR="/backup/logs"
mkdir -p $BACKUP_DIR

# 压缩日志
tar -czf $BACKUP_DIR/logs_$DATE.tar.gz /var/log/*.log /var/log/nginx/*.log

# 清理旧备份
find $BACKUP_DIR -name "*.tar.gz" -mtime +90 -delete

# 添加到 crontab
# 0 2 * * * /usr/local/bin/backup-logs.sh

# 1. 查看系统日志
journalctl -xe

# 2. 查看最近日志
journalctl -n 50

# 3. 查看错误日志
journalctl -p err -b

# 4. 实时跟踪
journalctl -f

# 1. 创建测试日志
mkdir -p /var/log/myapp
echo "test" > /var/log/myapp/test.log

# 2. 创建配置
sudo vim /etc/logrotate.d/myapp
/var/log/myapp/*.log {
    daily
    rotate 7
    compress
    missingok
    notifempty
}

# 3. 测试
sudo logrotate -d /etc/logrotate.d/myapp

# 4. 强制执行
sudo logrotate -f /etc/logrotate.d/myapp

# 5. 查看结果
ls -la /var/log/myapp/

# 1. 安装 goaccess
sudo apt install goaccess

# 2. 生成报告
sudo goaccess /var/log/nginx/access.log -o /var/www/html/report.html --log-format=COMBINED

# 3. 查看报告
# 访问 http://server/report.html