diff --git a/cucyuqing/cmd/es-sync.py b/cucyuqing/cmd/es-sync.py index 3eb2b1a..dd0de20 100644 --- a/cucyuqing/cmd/es-sync.py +++ b/cucyuqing/cmd/es-sync.py @@ -66,9 +66,13 @@ async def fetch(interval: ESInterval, size=1000) -> AsyncIterable[dict]: # 这里使用递归 if len(docs) == size: print("继续请求下一页数据") + start_time = parse_unixtime(docs[-1]["crawled_at"]) + if start_time == interval.end_time: + print("已经到达时间段末尾,停止请求", start_time) + return async for doc in fetch( ESInterval( - start_time=parse_unixtime(docs[-1]["crawled_at"]), + start_time=start_time, end_time=interval.end_time, ), size,