Elasticsearch索引生命周期管理 #

ILM作用
├── 自动化管理
│   ├── 自动滚动索引
│   ├── 自动调整设置
│   └── 自动删除过期数据
├── 数据分层
│   ├── 热数据层
│   ├── 温数据层
│   └── 冷数据层
└── 降低成本
    └── 自动迁移和删除

PUT /_ilm/policy/logs_policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_size": "50gb",
            "max_age": "1d"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "forcemerge": {
            "max_num_segments": 1
          },
          "shrink": {
            "number_of_shards": 1
          },
          "readonly": {},
          "set_priority": {
            "priority": 50
          }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "freeze": {},
          "set_priority": {
            "priority": 0
          }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

GET /_ilm/policy/logs_policy

GET /_ilm/policy

DELETE /_ilm/policy/logs_policy

{
  "hot": {
    "min_age": "0ms",
    "actions": {
      "rollover": {
        "max_size": "50gb",
        "max_age": "1d",
        "max_docs": 1000000000
      },
      "set_priority": {
        "priority": 100
      }
    }
  }
}

{
  "warm": {
    "min_age": "7d",
    "actions": {
      "forcemerge": {
        "max_num_segments": 1
      },
      "shrink": {
        "number_of_shards": 1
      },
      "readonly": {},
      "allocate": {
        "require": {
          "data": "warm"
        }
      },
      "set_priority": {
        "priority": 50
      }
    }
  }
}

{
  "cold": {
    "min_age": "30d",
    "actions": {
      "freeze": {},
      "allocate": {
        "require": {
          "data": "cold"
        }
      },
      "set_priority": {
        "priority": 0
      }
    }
  }
}

{
  "delete": {
    "min_age": "90d",
    "actions": {
      "delete": {}
    }
  }
}

PUT /_index_template/logs_template
{
  "index_patterns": ["logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 1,
      "lifecycle.name": "logs_policy",
      "lifecycle.rollover_alias": "logs"
    },
    "aliases": {
      "logs": {
        "is_write_index": true
      }
    }
  }
}

PUT /logs-000001
{
  "aliases": {
    "logs": {
      "is_write_index": true
    }
  }
}

POST /logs/_doc
{
  "message": "Log entry",
  "@timestamp": "2024-01-01T00:00:00Z"
}

node.roles: [data_hot, data_content]

node.roles: [data_warm, data_content]

node.roles: [data_cold, data_content]

node.attr.data: hot

node.attr.data: warm

node.attr.data: cold

{
  "allocate": {
    "require": {
      "data": "warm"
    }
  }
}

GET /_ilm/status

GET /logs-000001/_ilm/explain

{
  "indices": {
    "logs-000001": {
      "index": "logs-000001",
      "managed": true,
      "policy": "logs_policy",
      "phase": "hot",
      "action": "rollover",
      "step": "check-rollover-ready",
      "phase_time": 1704067200000,
      "action_time": 1704067200000,
      "step_time": 1704067200000
    }
  }
}

POST /logs-000001/_ilm/retry

POST /_ilm/start

POST /_ilm/stop

PUT /logs-000001/_settings
{
  "lifecycle.name": null
}

热温冷架构
┌─────────────────────────────────────────────┐
│                  Hot Layer                   │
│  ├── 高性能存储(SSD)                         │
│  ├── 频繁写入和查询                          │
│  └── 多副本                                  │
├─────────────────────────────────────────────┤
│                 Warm Layer                   │
│  ├── 普通存储(HDD)                           │
│  ├── 只读查询                                │
│  └── 减少副本                                │
├─────────────────────────────────────────────┤
│                 Cold Layer                   │
│  ├── 归档存储                                │
│  ├── 偶尔查询                                │
│  └── 冻结索引                                │
└─────────────────────────────────────────────┘

PUT /_ilm/policy/hot_warm_cold_policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_size": "50gb",
            "max_age": "1d"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "forcemerge": {
            "max_num_segments": 1
          },
          "shrink": {
            "number_of_shards": 1
          },
          "allocate": {
            "require": {
              "data": "warm"
            }
          },
          "readonly": {},
          "set_priority": {
            "priority": 50
          }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "freeze": {},
          "allocate": {
            "require": {
              "data": "cold"
            }
          },
          "set_priority": {
            "priority": 0
          }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

策略设计建议
├── 合理设置阶段时间
│   └── 根据数据访问模式
├── 合理设置滚动条件
│   └── 避免过小或过大
├── 使用forcemerge
│   └── 减少段数量
└── 使用shrink
    └── 减少分片数量

监控建议
├── 监控ILM执行状态
├── 监控索引大小
├── 监控存储空间
└── 设置告警