mathdatech/python_reference
1
0
Fork 0
mirror of https://github.com/rasbt/python_reference.git synced 2024-11-23 20:11:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

updated readme

Browse Source
This commit is contained in:
rasbt 2015-02-06 12:01:46 -05:00
parent edc17e2907
commit 45c931b3f9
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
useful_scripts/large_csv_to_sqlite.py
View File

@ -17,6 +17,9 @@ out_sqlite = '../data/my.sqlite'
table_name = 'my_table' # name for the SQLite database table table_name = 'my_table' # name for the SQLite database table
chunksize = 100000 # number of lines to process at each iteration chunksize = 100000 # number of lines to process at each iteration
# columns that should be read from the CSV file
columns = ['molecule_id','charge','db','drugsnow','hba','hbd','loc','nrb','smiles']
# Get number of lines in the CSV file # Get number of lines in the CSV file
nlines = subprocess.check_output('wc -l %s' % in_csv, shell=True) nlines = subprocess.check_output('wc -l %s' % in_csv, shell=True)
nlines = int(nlines.split()[0]) nlines = int(nlines.split()[0])
@ -33,7 +36,7 @@ for i in range(0, nlines, chunksize):
skiprows=i) # skip rows that were already read skiprows=i) # skip rows that were already read
# columns to read # columns to read
df.columns = ['molecule_id','charge','db','drugsnow','hba','hbd','loc','nrb','smiles'] df.columns = columns
sql.to_sql(df, sql.to_sql(df,
name=table_name, name=table_name,