This bit of code will transfer a whole bunch of csv files in the ‘logs/’ directory into a single SQLite db file called
import pandas as pd import re import os import sqlite3 as sql # create list of all the different files list_files = [x for x in os.listdir('logs') if x.endswith('.csv')] # create a new file with a connection db_con = sql.connect('repo_data.db') # for 8-bit string instead of unicode...why? db_con.text_factory = str # now go through the things and create a table for each for str_raw_name in list_files: # clean name and lower str_name = re.sub('.csv', '', str_raw_name).lower() # open the file and create df = pd.read_csv('logs/%s' % str_raw_name) # pass on to the database df.to_sql(str_name, con = db_con, index=False) print 'DONE', str_name
NOTE: I kept getting an error about some 8-bit blah blah blah text thing. I googled it and fixed it (line 13), but I still have no idea what it means.