heimoshuiyu/cucyuqing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

从ES获取的结果中过滤掉\x00

Browse Source
This commit is contained in:
黑墨水鱼
2024-09-20 15:20:19 +08:00
parent 43ddd8665c
commit fff2a32d7e
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cucyuqing/cmd/es-sync.py
View File

@@ -66,6 +66,8 @@ async def fetch(interval: ESInterval, size=1000) -> AsyncIterable[dict]:
f'用时 {int(duration)} 秒，获取到 {len(docs)} 条数据，最早时间 {parse_unixtime(docs[0]["crawled_at"])}，最晚时间 {parse_unixtime(docs[-1]["crawled_at"])}' f'用时 {int(duration)} 秒，获取到 {len(docs)} 条数据，最早时间 {parse_unixtime(docs[0]["crawled_at"])}，最晚时间 {parse_unixtime(docs[-1]["crawled_at"])}'
) )
for d in docs: for d in docs:
d['title'] = d['title'].replace('\x00', '')
d['content'] = d['content'].replace('\x00', '')
yield d yield d
# 如果当前时间度的数据量 = size 说明还有数据，继续请求 # 如果当前时间度的数据量 = size 说明还有数据，继续请求
# 这里使用递归 # 这里使用递归