LogReader: add tqdm back to run_across_segments (#31216)

tqdm
pull/31217/head
Justin Newberry 2 years ago committed by GitHub
parent ed7cfc2343
commit 7835f9cce0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 4
      tools/lib/logreader.py

@ -9,6 +9,7 @@ import os
import pathlib import pathlib
import re import re
import sys import sys
import tqdm
import urllib.parse import urllib.parse
import warnings import warnings
@ -259,7 +260,8 @@ class LogReader:
def run_across_segments(self, num_processes, func): def run_across_segments(self, num_processes, func):
with multiprocessing.Pool(num_processes) as pool: with multiprocessing.Pool(num_processes) as pool:
ret = [] ret = []
for p in pool.map(partial(self._run_on_segment, func), range(len(self.logreader_identifiers))): num_segs = len(self.logreader_identifiers)
for p in tqdm.tqdm(pool.imap(partial(self._run_on_segment, func), range(num_segs)), total=num_segs):
ret.extend(p) ret.extend(p)
return ret return ret

Loading…
Cancel
Save