|
|
|
|
|
from elasticsearch import Elasticsearch
|
|
|
|
|
|
import json
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 如有报错：Result window is too large, from + size must be less than or equal to: [10000]
|
|
|
|
|
|
# 执行以下修改：【不再使用此方式，防止内存溢出，使用如下的scroll api处理】
|
|
|
|
|
|
# curl -XPUT "http://192.168.2.15:9200/props_change_log/_settings" -d '{ "index" : { "max_result_window" : 1000000 } }'
|
|
|
|
|
|
|
|
|
|
|
|
# 定义数据写入的文件路径
|
|
|
|
|
|
root_path = "D:/xxx.json"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def record_docs(root_path, record):
|
|
|
|
|
|
with open(root_path, "a", encoding="utf-8") as file:
|
|
|
|
|
|
file.write(record)
|
|
|
|
|
|
file.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 定义配置
|
|
|
|
|
|
host = "192.168.2.15:9200"
|
|
|
|
|
|
# index = "props_change_log"
|
|
|
|
|
|
index = "time_limited_props_log"
|
|
|
|
|
|
scroll = "1m"
|
|
|
|
|
|
size = 1000
|
|
|
|
|
|
body = {
|
|
|
|
|
|
"query": {"match_all": {}},
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
es = Elasticsearch(hosts=host)
|
|
|
|
|
|
|
|
|
|
|
|
# es.indices.refresh(index="by_petskill_log")
|
|
|
|
|
|
|
|
|
|
|
|
# 利用json.dumps处理hits数据,将返回str类型
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def process_hits(hits):
|
|
|
|
|
|
for item in hits:
|
|
|
|
|
|
# print(json.dumps(item["_source"], indent=2))
|
|
|
|
|
|
# print(json.dumps(item["_source"]))
|
|
|
|
|
|
record_docs(root_path, json.dumps(item["_source"]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 检查index是否存在
|
|
|
|
|
|
if not es.indices.exists(index=index):
|
|
|
|
|
|
print("index：" + index + "不存在")
|
|
|
|
|
|
exit()
|
|
|
|
|
|
|
|
|
|
|
|
# 数据总量大于等于1000时，用scroll api搜索
|
|
|
|
|
|
data_scroll = es.search(index=index, scroll=scroll, size=size, body=body)
|
|
|
|
|
|
|
|
|
|
|
|
# 数据总量少于1000时，将使用此搜索方式
|
|
|
|
|
|
data = es.search(index=index, size=size, body=body)
|
|
|
|
|
|
|
|
|
|
|
|
# 获取scroll id
|
|
|
|
|
|
scroll_id = data_scroll["_scroll_id"]
|
|
|
|
|
|
|
|
|
|
|
|
# 获取匹配文档数
|
|
|
|
|
|
# scroll_size = len(data["hits"]["hits"])
|
|
|
|
|
|
# 获取匹配文档总数
|
|
|
|
|
|
scroll_size = data["hits"]["total"]
|
|
|
|
|
|
print("匹配到文档总数为：" + str(scroll_size) + "\n")
|
|
|
|
|
|
|
|
|
|
|
|
if scroll_size > 0 and scroll_size < 1000:
|
|
|
|
|
|
process_hits(data["hits"]["hits"])
|
|
|
|
|
|
elif scroll_size >= 1000:
|
|
|
|
|
|
while scroll_size > 0:
|
|
|
|
|
|
# print("处理中ing......")
|
|
|
|
|
|
# 滚动处理之前，先处理当前匹配项
|
|
|
|
|
|
process_hits(data_scroll["hits"]["hits"])
|
|
|
|
|
|
data_scroll = es.scroll(scroll_id=scroll_id, scroll=scroll)
|
|
|
|
|
|
|
|
|
|
|
|
# 更新scroll id
|
|
|
|
|
|
scroll_id = data_scroll["_scroll_id"]
|
|
|
|
|
|
# 获取最后一次scroll的数量
|
|
|
|
|
|
scroll_size = len(data_scroll["hits"]["hits"])
|
