From fe21bc48389271a434c2a548e80dfa7e32112b29 Mon Sep 17 00:00:00 2001 From: rasbt Date: Sat, 26 Apr 2014 04:11:20 -0400 Subject: [PATCH] copy find appr --- .../timeit_tests-checkpoint.ipynb | 82 +++++++++++++++---- benchmarks/timeit_tests.ipynb | 82 +++++++++++++++---- 2 files changed, 128 insertions(+), 36 deletions(-) diff --git a/.ipynb_checkpoints/timeit_tests-checkpoint.ipynb b/.ipynb_checkpoints/timeit_tests-checkpoint.ipynb index 3e39ece..28e49fa 100644 --- a/.ipynb_checkpoints/timeit_tests-checkpoint.ipynb +++ b/.ipynb_checkpoints/timeit_tests-checkpoint.ipynb @@ -1,7 +1,7 @@ { "metadata": { "name": "", - "signature": "sha256:b8d5bf725bbe6b0e07d0b544cb5a65cdce7b4c162fd421985a4aa25ea57f9151" + "signature": "sha256:9e62469f6250ac6d58e6d3d2c67a5995d1505778b147ae918d71f4ebc1c12fb6" }, "nbformat": 3, "nbformat_minor": 0, @@ -1270,14 +1270,14 @@ "import subprocess\n", "\n", "def subprocess_findcopy(path, search_str, dest): \n", - " query = 'find %s -name \"%s\" -exec cp {}\" %s \\;' %(path, search_str, dest)\n", + " query = 'find %s -name \"%s\" -exec cp {} %s \\;' %(path, search_str, dest)\n", " subprocess.call(query, shell=True)\n", " return " ], "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 2 + "prompt_number": 30 }, { "cell_type": "markdown", @@ -1297,12 +1297,16 @@ "def walk_findcopy(path, search_str, dest):\n", " for path, subdirs, files in os.walk(path):\n", " for name in fnmatch.filter(files, search_str):\n", - " shutil.copy(os.path.join(path,name), dest)" + " try:\n", + " shutil.copy(os.path.join(path,name), dest)\n", + " except NameError:\n", + " pass\n", + " return" ], "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 3 + "prompt_number": 33 }, { "cell_type": "code", @@ -1310,19 +1314,33 @@ "input": [ "import timeit\n", "\n", + "\n", + "def findcopy_timeit(inpath, outpath, search_str):\n", + " \n", + " shutil.rmtree(outpath)\n", + " os.mkdir(outpath)\n", + " print(50*'#')\n", + " print('subprocsess call')\n", + " %timeit subprocess_findcopy(inpath, search_str, outpath)\n", + " print(\"copied %s files\" %len(os.listdir(outpath)))\n", + " shutil.rmtree(outpath)\n", + " os.mkdir(outpath)\n", + " print('\\nos.walk approach')\n", + " %timeit walk_findcopy(inpath, search_str, outpath)\n", + " print(\"copied %s files\" %len(os.listdir(outpath)))\n", + " print(50*'#')\n", + "\n", "print('small tree')\n", "inpath = '/Users/sebastian/Desktop/testdir_in'\n", "outpath = '/Users/sebastian/Desktop/testdir_out'\n", - "searchstr = '*.png'\n", - "%timeit subprocess_findcopy(inpath, searchstr, outpath)\n", - "%timeit walk_findcopy(inpath, searchstr, outpath)\n", + "search_str = '*.png'\n", + "findcopy_timeit(inpath, outpath, search_str)\n", "\n", - "print('\\nlarger tree')\n", + "print('larger tree')\n", "inpath = '/Users/sebastian/Dropbox'\n", "outpath = '/Users/sebastian/Desktop/testdir_out'\n", - "searchstr = '*.csv'\n", - "%timeit subprocess_findcopy(inpath, searchstr, outpath)\n", - "%timeit walk_findcopy(inpath, searchstr, outpath)" + "search_str = '*.csv'\n", + "findcopy_timeit(inpath, outpath, search_str)\n" ], "language": "python", "metadata": {}, @@ -1332,7 +1350,7 @@ "stream": "stdout", "text": [ "small tree\n", - "100 loops, best of 3: 8.48 ms per loop" + "##################################################" ] }, { @@ -1340,7 +1358,8 @@ "stream": "stdout", "text": [ "\n", - "100 loops, best of 3: 22.3 ms per loop" + "subprocsess call\n", + "1 loops, best of 3: 268 ms per loop" ] }, { @@ -1348,9 +1367,23 @@ "stream": "stdout", "text": [ "\n", + "copied 13 files\n", "\n", + "os.walk approach\n", + "100 loops, best of 3: 12.2 ms per loop" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "copied 13 files\n", + "##################################################\n", "larger tree\n", - "10 loops, best of 3: 7.13 ms per loop" + "##################################################\n", + "subprocsess call\n", + "1 loops, best of 3: 623 ms per loop" ] }, { @@ -1358,18 +1391,31 @@ "stream": "stdout", "text": [ "\n", - "1 loops, best of 3: 413 ms per loop" + "copied 105 files\n", + "\n", + "os.walk approach\n", + "1 loops, best of 3: 417 ms per loop" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "\n" + "\n", + "copied 105 files\n", + "##################################################\n" ] } ], - "prompt_number": 4 + "prompt_number": 35 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] }, { "cell_type": "markdown", diff --git a/benchmarks/timeit_tests.ipynb b/benchmarks/timeit_tests.ipynb index 3e39ece..28e49fa 100644 --- a/benchmarks/timeit_tests.ipynb +++ b/benchmarks/timeit_tests.ipynb @@ -1,7 +1,7 @@ { "metadata": { "name": "", - "signature": "sha256:b8d5bf725bbe6b0e07d0b544cb5a65cdce7b4c162fd421985a4aa25ea57f9151" + "signature": "sha256:9e62469f6250ac6d58e6d3d2c67a5995d1505778b147ae918d71f4ebc1c12fb6" }, "nbformat": 3, "nbformat_minor": 0, @@ -1270,14 +1270,14 @@ "import subprocess\n", "\n", "def subprocess_findcopy(path, search_str, dest): \n", - " query = 'find %s -name \"%s\" -exec cp {}\" %s \\;' %(path, search_str, dest)\n", + " query = 'find %s -name \"%s\" -exec cp {} %s \\;' %(path, search_str, dest)\n", " subprocess.call(query, shell=True)\n", " return " ], "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 2 + "prompt_number": 30 }, { "cell_type": "markdown", @@ -1297,12 +1297,16 @@ "def walk_findcopy(path, search_str, dest):\n", " for path, subdirs, files in os.walk(path):\n", " for name in fnmatch.filter(files, search_str):\n", - " shutil.copy(os.path.join(path,name), dest)" + " try:\n", + " shutil.copy(os.path.join(path,name), dest)\n", + " except NameError:\n", + " pass\n", + " return" ], "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 3 + "prompt_number": 33 }, { "cell_type": "code", @@ -1310,19 +1314,33 @@ "input": [ "import timeit\n", "\n", + "\n", + "def findcopy_timeit(inpath, outpath, search_str):\n", + " \n", + " shutil.rmtree(outpath)\n", + " os.mkdir(outpath)\n", + " print(50*'#')\n", + " print('subprocsess call')\n", + " %timeit subprocess_findcopy(inpath, search_str, outpath)\n", + " print(\"copied %s files\" %len(os.listdir(outpath)))\n", + " shutil.rmtree(outpath)\n", + " os.mkdir(outpath)\n", + " print('\\nos.walk approach')\n", + " %timeit walk_findcopy(inpath, search_str, outpath)\n", + " print(\"copied %s files\" %len(os.listdir(outpath)))\n", + " print(50*'#')\n", + "\n", "print('small tree')\n", "inpath = '/Users/sebastian/Desktop/testdir_in'\n", "outpath = '/Users/sebastian/Desktop/testdir_out'\n", - "searchstr = '*.png'\n", - "%timeit subprocess_findcopy(inpath, searchstr, outpath)\n", - "%timeit walk_findcopy(inpath, searchstr, outpath)\n", + "search_str = '*.png'\n", + "findcopy_timeit(inpath, outpath, search_str)\n", "\n", - "print('\\nlarger tree')\n", + "print('larger tree')\n", "inpath = '/Users/sebastian/Dropbox'\n", "outpath = '/Users/sebastian/Desktop/testdir_out'\n", - "searchstr = '*.csv'\n", - "%timeit subprocess_findcopy(inpath, searchstr, outpath)\n", - "%timeit walk_findcopy(inpath, searchstr, outpath)" + "search_str = '*.csv'\n", + "findcopy_timeit(inpath, outpath, search_str)\n" ], "language": "python", "metadata": {}, @@ -1332,7 +1350,7 @@ "stream": "stdout", "text": [ "small tree\n", - "100 loops, best of 3: 8.48 ms per loop" + "##################################################" ] }, { @@ -1340,7 +1358,8 @@ "stream": "stdout", "text": [ "\n", - "100 loops, best of 3: 22.3 ms per loop" + "subprocsess call\n", + "1 loops, best of 3: 268 ms per loop" ] }, { @@ -1348,9 +1367,23 @@ "stream": "stdout", "text": [ "\n", + "copied 13 files\n", "\n", + "os.walk approach\n", + "100 loops, best of 3: 12.2 ms per loop" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "copied 13 files\n", + "##################################################\n", "larger tree\n", - "10 loops, best of 3: 7.13 ms per loop" + "##################################################\n", + "subprocsess call\n", + "1 loops, best of 3: 623 ms per loop" ] }, { @@ -1358,18 +1391,31 @@ "stream": "stdout", "text": [ "\n", - "1 loops, best of 3: 413 ms per loop" + "copied 105 files\n", + "\n", + "os.walk approach\n", + "1 loops, best of 3: 417 ms per loop" ] }, { "output_type": "stream", "stream": "stdout", "text": [ - "\n" + "\n", + "copied 105 files\n", + "##################################################\n" ] } ], - "prompt_number": 4 + "prompt_number": 35 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] }, { "cell_type": "markdown",