diff --git a/useful_scripts/conc_gzip_files.py b/useful_scripts/conc_gzip_files.py index edc5c4c..da849c9 100644 --- a/useful_scripts/conc_gzip_files.py +++ b/useful_scripts/conc_gzip_files.py @@ -1,9 +1,12 @@ # Sebastian Raschka 03/2014 import gzip +import shutil import os -def conc_gzip_files(in_dir, out_file, append=False): +#import pyprind + +def conc_gzip_files(in_dir, out_file, append=False, print_progress=True): """ Reads contents from gzipped ASCII or UTF-8 files, decodes them, and appends the lines to one output file. @@ -12,17 +15,19 @@ def conc_gzip_files(in_dir, out_file, append=False): out_file (str): Path to the resulting file append (bool): If true, it appends contents to an exisiting file, else creates a new output file. + print_progress (bool): prints progress bar if true. """ write_mode = 'wb' - if append: - write_mode = 'ab' gzips = [os.path.join(in_dir, i) for i in os.listdir(in_dir) if i.endswith('.gz')] - with open(out_file, write_mode) as ofile: + #if print_progress: + # pbar = pyprind.ProgBar(len(gzips)) + with open(out_file, 'ab' if append else 'wb') as ofile: for f in gzips: with gzip.open(f, 'rb') as gzipf: - for line in gzipf: - ofile.write(line) + shutil.copyfileobj(gzipf, ofile) + #if print_progress: + # pbar.update() if __name__ == '__main__': conc_gzip_files('/home/usr/my_dir', '/home/usr/test.txt')