本文主要測試驗證 Elasticsearch 各版本快照在 Easysearch 中進(jìn)行數(shù)據(jù)恢復(fù)。

準(zhǔn)備測試數(shù)據(jù)

索引

別名

模版

生命周期策略

創(chuàng)建快照

PUT /_snapshot/my_backup
{
  "type": "fs",
  "settings": {
    "location": "/infini/test/es_backup"
  }
}

PUT /_snapshot/my_backup/snapshot_1
{
  "indices": "*",
  "ignore_unavailable": false,
  "include_global_state": false
}

GET /_snapshot/my_backup/snapshot_1

• ignore_unavailable：如果 indices 列表中的索引不存在躺屁，則是否忽略該索引而不是使快照失敗冤今。默認(rèn)值為 false 。
• include_global_state：是否在快照中包含集群狀態(tài)（包括索引模版桥状、生命周期配置帽揪、持久化配置等）。默認(rèn)值為 true 辅斟，建議設(shè)為 false转晰。

恢復(fù)快照

POST /_snapshot/my_backup/snapshot_1/_restore
{
  "indices": "*",
  "ignore_unavailable": false,
  "include_global_state": false,
  "include_aliases": true,
  "ignore_index_settings": [
    "index.lifecycle.indexing_complete"
  ]
}

• ignore_unavailable：如果 indices 列表中的索引不存在，則是否忽略該索引而不是使還原操作失敗。默認(rèn)值為 false 查邢。
• include_global_state：是否還原群集狀態(tài)蔗崎。默認(rèn)值為 false 。
• include_aliases：是否恢復(fù)別名及其關(guān)聯(lián)索引扰藕。默認(rèn)值為 true 缓苛。
• index.lifecycle.indexing_complete 配置不支持，忽略掉邓深。

數(shù)據(jù)驗證

索引

通過 gateway 進(jìn)行數(shù)據(jù)比對

path.data: data
path.logs: log

#show progress bar
#progress_bar.enabled: true

elasticsearch:
  - name: source
    enabled: true
    endpoints:
      - http://192.168.3.185:29200
  - name: target
    enabled: true
    endpoints:
      - https://192.168.3.185:9205
    basic_auth:
      username: admin
      password: admin

pipeline:
  - name: index_diff_service
    auto_start: true
    processor:
      - dag:
          mode: wait_all
          parallel:
            - dump_hash: #dump es1's doc
                sort_document_fields: true
                indices: ".infini_activities-000004"   ##需要比對的索引名
                scroll_time: "10m"
                elasticsearch: "source"
                #              query_string: "_id:c8es70pu46lgfdgmja9g-1646117763293610802-2"
                #              fields: "doc_hash"
                output_queue: "source_docs"
                batch_size: 5000
                slice_size: 1
            #              hash_func: "xxhash64"
            - dump_hash: #dump es2's doc
                indices: ".infini_activities-000004"
                scroll_time: "10m"
                #              fields: "doc_hash"
                #              query_string: "_id:c8es70pu46lgfdgmja9g-1646117763293610802-2"
                batch_size: 5000
                slice_size: 1
                #              hash_func: "xxhash64"
                elasticsearch: "target"
                output_queue: "target_docs"
          end:
            - index_diff:
                diff_queue: "diff_result"
                buffer_size: 10
                text_report: true #如果要存 es未桥，這個開關(guān)關(guān)閉，開啟 pipeline 的 diff_result_ingest 任務(wù)
                source_queue: "source_docs"
                target_queue: "target_docs"
#pipeline:
#  - name: diff_result_ingest
#    processor:
#      - json_indexing:
#          index_name: "diff_result"
#          elasticsearch: "source"
#          input_queue: "diff_result"

./gateway-linux-amd64 -config data_check.yml

別名

模版

PUT _template/.infini_activities-rollover
{
  "order": 100000,
  "index_patterns": [
    ".infini_activities*"
  ],
  "settings": {
    "index": {
      "format": "7",
      "lifecycle": {
        "name": "ilm_.infini_metrics-30days-retention",
        "rollover_alias": ".infini_activities"
      },
      "codec": "best_compression",
      "number_of_shards": "1",
      "translog": {
        "durability": "async"
      }
    }
  },
  "mappings": {
    "dynamic_templates": [
      {
        "strings": {
          "mapping": {
            "ignore_above": 256,
            "type": "keyword"
          },
          "match_mapping_type": "string"
        }
      }
    ]
  },
  "aliases": {}
}

PUT _template/.infini
{
  "order": 0,
  "index_patterns": [
    ".infini_*"
  ],
  "settings": {
    "index": {
      "max_result_window": "10000000",
      "mapping": {
        "total_fields": {
          "limit": "20000"
        }
      },
      "analysis": {
        "analyzer": {
          "suggest_text_search": {
            "filter": [
              "word_delimiter"
            ],
            "tokenizer": "classic"
          }
        }
      },
      "number_of_shards": "1"
    }
  },
  "mappings": {
    "dynamic_templates": [
      {
        "strings": {
          "mapping": {
            "ignore_above": 256,
            "type": "keyword"
          },
          "match_mapping_type": "string"
        }
      }
    ]
  },
  "aliases": {}
}

生命周期策略

PUT _ilm/policy/ilm_.infini_metrics-30days-retention
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_size": "50gb",
            "max_age": "30d"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "delete": {
        "min_age": "30d",
        "actions": {
          "delete": {
          }
        }
      }
    }
  }
}

注：不支持 "delete_searchable_snapshot": true 配置

測試結(jié)果

關(guān)于 Easysearch

about easysearch

INFINI Easysearch 是一個分布式的近實時搜索與分析引擎懂讯，核心引擎基于開源的 Apache Lucene。 Easysearch 衍生自基于開源協(xié)議 Apache 2.0 的 Elasticsearch 7.10 版本台颠。 Easysearch 的目標(biāo)是提供一個輕量級的 Elasticsearch 可替代版本褐望，并繼續(xù)完善和支持更多的企業(yè)級功能。 與 Elasticsearch 相比串前，Easysearch 更關(guān)注在搜索業(yè)務(wù)場景的優(yōu)化和繼續(xù)保持其產(chǎn)品的簡潔與易用性瘫里。

詳情參見：官方文檔