diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..40f60288 Binary files /dev/null and b/.DS_Store differ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..2196d1af --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/Users/noahpovis/opt/anaconda3/envs/unit3lambda/bin/python" +} \ No newline at end of file diff --git a/demo.sqlite b/demo.sqlite new file mode 100644 index 00000000..9761c261 Binary files /dev/null and b/demo.sqlite differ diff --git a/demo.sqlite3 b/demo.sqlite3 new file mode 100644 index 00000000..d48ba1eb Binary files /dev/null and b/demo.sqlite3 differ diff --git a/module1-introduction-to-sql/buddymove_holidayiq.db b/module1-introduction-to-sql/buddymove_holidayiq.db new file mode 100644 index 00000000..73b2146c Binary files /dev/null and b/module1-introduction-to-sql/buddymove_holidayiq.db differ diff --git a/module1-introduction-to-sql/buddymove_holidayiq.py b/module1-introduction-to-sql/buddymove_holidayiq.py new file mode 100644 index 00000000..14774fad --- /dev/null +++ b/module1-introduction-to-sql/buddymove_holidayiq.py @@ -0,0 +1,40 @@ + +import pandas as pd +import sqlite3 + +conn = sqlite3.connect('buddymove_holidayiq.sqlite3') +conn + +cursor = conn.cursor() +# df = pd.read_csv('/Users/noahpovis/Desktop/Lambda Clones/DS-Unit-3-Sprint-2-SQL-and-Databases/module1-introduction-to-sql/buddymove_holidayiq.csv') +# df.to_sql('buddymove_holidayiq' , con=conn) + +# - Count how many rows you have - it should be 249! +query = ''' + SELECT + COUNT(*) + FROM buddymove_holidayiq + ''' +result = cursor.execute(query).fetchall() +print(f'Number of rows are {result[0][0]}') +# - How many users who reviewed at least 100 `Nature` in the category also +# reviewed at least 100 in the `Shopping` category? +query = ''' + SELECT + COUNT(*) + FROM buddymove_holidayiq + WHERE buddymove_holidayiq.Nature >= 100 AND buddymove_holidayiq.Shopping >= 100 + ''' +result = cursor.execute(query).fetchall() +print(f'{result[0][0]} users reviewed both Nature and Shopping categories over 100 times') +# - (*Stretch*) What are the average number of reviews for each category? +query = ''' +SELECT +round(AVG(Nature),2) as Nature_Avg, round(AVG(Sports),2) as Sport_Avg, round(AVG(Religious),2) as Religion_Avg, +round(AVG(Theatre),2) as Theatre_Avg, round(AVG(Shopping) + , 2) as Shopping_avg, round(AVG(Picnic),2) as Picnic_AVG +FROM (SELECT * FROM buddymove_holidayiq) +''' +results = cursor.execute(query).fetchall() + +print(results) diff --git a/module1-introduction-to-sql/buddymove_holidayiq.sqlite3 b/module1-introduction-to-sql/buddymove_holidayiq.sqlite3 new file mode 100644 index 00000000..417b5841 Binary files /dev/null and b/module1-introduction-to-sql/buddymove_holidayiq.sqlite3 differ diff --git a/module1-introduction-to-sql/practice_table.db b/module1-introduction-to-sql/practice_table.db new file mode 100644 index 00000000..73b2146c Binary files /dev/null and b/module1-introduction-to-sql/practice_table.db differ diff --git a/module1-introduction-to-sql/rpg_db.py b/module1-introduction-to-sql/rpg_db.py new file mode 100644 index 00000000..8d82fe92 --- /dev/null +++ b/module1-introduction-to-sql/rpg_db.py @@ -0,0 +1,108 @@ +"""- How many total Characters are there? +- How many of each specific subclass? +- How many total Items? +- How many of the Items are weapons? How many are not? +- How many Items does each character have? (Return first 20 rows) +- How many Weapons does each character have? (Return first 20 rows) +- On average, how many Items does each Character have? +- On average, how many Weapons does each character have? +""" + + +import sqlite3 + +conn = sqlite3.connect('rpg_db.sqlite3') +curs = conn.cursor() + + +#How many total Characters are there? +count_characters = 'SELECT COUNT(*) FROM charactercreator_character;' +print(curs.execute(count_characters).fetchall()) + +# How many of each specific subclass? +count_cleric = 'SELECT COUNT(*) FROM charactercreator_cleric;' +count_fighter = 'SELECT COUNT(*) FROM charactercreator_fighter;' +count_mage = 'SELECT COUNT(*) FROM charactercreator_mage;' +count_necromancer = 'SELECT COUNT(*) FROM charactercreator_necromancer;' +count_thief = 'SELECT COUNT(*) FROM charactercreator_thief;' + +print('Cleric number:', curs.execute(count_cleric).fetchall()) +print('Fighter number:', curs.execute(count_fighter).fetchall()) +print('Mage number:', curs.execute(count_mage).fetchall()) +print('Necromancer number:', curs.execute(count_necromancer).fetchall()) +print('Thief number:', curs.execute(count_thief).fetchall()) + +#How many total Items? +count_armory = 'SELECT COUNT(*) FROM armory_item;' +print('Item count:', curs.execute(count_armory).fetchall()) + +#How many of the Items are weapons? How many are not? +count_weapons = 'SELECT COUNT(DISTINCT armory_item.item_id) FROM armory_item INNER JOIN armory_weapon ON armory_item.item_id = armory_weapon.item_ptr_id;' +print('Total weapon count in items:', curs.execute(count_weapons).fetchall()) + +count_not_weapons = 'SELECT COUNT(*) FROM armory_item, armory_weapon WHERE armory_item.item_id <> armory_weapon.item_ptr_id;' +print('Total item count in items that are not weapons', curs.execute(count_not_weapons).fetchall()[0][0]) + +#How many Items does each character have? (Return first 20 rows) +# item_cleric = 'SELECT COUNT(*) FROM charactercreator_character_inventory, charactercreator_cleric WHERE charactercreator_cleric.character_ptr_id = charactercreator_character_inventory.character_id LIMIT 20;' +# print('Cleric item count:', curs.execute(item_cleric).fetchall()[0][0]) + +# item_fighter = 'SELECT COUNT(*) FROM charactercreator_character_inventory, charactercreator_fighter WHERE charactercreator_fighter.character_ptr_id = charactercreator_character_inventory.character_id;' +# print('Fighter item count:', curs.execute(item_fighter).fetchall()[0][0]) + +# item_mage = 'SELECT COUNT(*) FROM charactercreator_character_inventory, charactercreator_mage WHERE charactercreator_mage.character_ptr_id = charactercreator_character_inventory.character_id;' +# print('Mage item count:', curs.execute(item_mage).fetchall()[0][0]) + +# item_necro = 'SELECT COUNT(*) FROM charactercreator_character_inventory, charactercreator_necromancer WHERE charactercreator_necromancer.mage_ptr_id = charactercreator_character_inventory.character_id;' +# print('Necromancer item count:', curs.execute(item_necro).fetchall()[0][0]) + +# item_thief = 'SELECT COUNT(*) FROM charactercreator_character_inventory, charactercreator_thief WHERE charactercreator_thief.character_ptr_id = charactercreator_character_inventory.character_id;' +# print('Thief item count:', curs.execute(item_thief).fetchall()[0][0]) + +query = 'SELECT character_id, COUNT(distinct item_id) FROM charactercreator_character_inventory GROUP BY character_id LIMIT 20' +result = curs.execute(query).fetchall() +print('\n Each character has the following number of items:') +for each in result: + print(f'Character {each[0]} has {each[1]} item(s).') + +# How many Weapons does each character have? (Return first 20 rows) +query = ''' + SELECT + charactercreator_character_inventory.character_id, + count(distinct item_ptr_id) + FROM + charactercreator_character_inventory + LEFT JOIN armory_weapon ON armory_weapon.item_ptr_id = charactercreator_character_inventory.item_id + GROUP BY + character_id + LIMIT + 20 +''' +result = curs.execute(query).fetchall() +print('\nEach character has the following number of weapons:') +for each in result: + print(f'Character {each[0]} has {each[1]} weapon(s).') + +# - On average, how many Items does each Character have? +query = ''' + SELECT + COUNT(item_id) as a, COUNT(distinct character_id) as b + FROM + charactercreator_character_inventory + ''' +result = curs.execute(query).fetchall() +print(f'\nOn average, each character has {result [0][0] / result [0][1]} items.') + +# On average, how many Weapons does each character have? + +query = ''' + SELECT + COUNT(item_ptr_id) as a, count(distinct character_id) as b + FROM + charactercreator_character_inventory + LEFT JOIN armory_weapon ON armory_weapon.item_ptr_id = charactercreator_character_inventory.item_id +''' +result = curs.execute(query).fetchall() +print(f'\nOne average, each character has {result[0][0] / result[0][1]} weapons\n') + + diff --git a/module2-sql-for-analysis/insert_titanic.py b/module2-sql-for-analysis/insert_titanic.py new file mode 100644 index 00000000..698b4ded --- /dev/null +++ b/module2-sql-for-analysis/insert_titanic.py @@ -0,0 +1,67 @@ +# Then, set up a new table for the Titanic data (`titanic.csv`) - spend some time +# thinking about the schema to make sure it is appropriate for the columns. +# [Enumerated types](https://www.postgresql.org/docs/9.1/datatype-enum.html) may +# be useful. Once it is set up, write a `insert_titanic.py` script that uses +# `psycopg2` to connect to and upload the data from the csv, and add the file to +# your repo. Then start writing PostgreSQL queries to explore the data! + +import sqlite3 +import psycopg2 +import pandas as pd + +#read in csv +df = pd.read_csv('titanic.csv') +df['Name'] = df['Name'].str.replace("'"," ") + +df.info() + +#create sqlite3 +conn = sqlite3.connect('titanic.sqlite3') +#create cursor +curs = conn.cursor() +#convert to sql +df.to_sql('titanic', conn) + +#pulling out data from the sql base we just made +get_titanic = 'SELECT * FROM titanic;' +passangers = curs.execute(get_titanic).fetchall() + +#put info and connect elephant +dbname = 'njskuvoi' +user = 'njskuvoi' +password = 'V4hoTmOL2IlAb_98ylcdRtErA_ERzFMo' +host = 'ruby.db.elephantsql.com' +pg_conn = psycopg2.connect(dbname=dbname, user=user, + password=password, host=host) + +create_table_statement = """ + +CREATE TABLE titanic1 ( + id SERIAL PRIMARY KEY, + Survived INTEGER, + pclass INTEGER, + name VARCHAR (140), + sex VARCHAR (10), + age FLOAT(1), + siblings_spouses_aboard INTEGER, + parents_children_aboard INTEGER, + fare FLOAT(4) +); +""" +#CREATE NEW CURSOR +pg_curs = pg_conn.cursor() + +#EXECUTE +pg_curs.execute(create_table_statement) + +for x in passangers: + insert_passanger = """ + INSERT INTO titanic1 + (Survived, Pclass, Name, Sex, Age, + Siblings_Spouses_Aboard, Parents_Children_Aboard, Fare) + VALUES """ + str(x[1:]) + ";" + pg_curs.execute(insert_passanger) + + +pg_conn.commit() + diff --git a/module2-sql-for-analysis/part1.py b/module2-sql-for-analysis/part1.py new file mode 100644 index 00000000..53345e85 --- /dev/null +++ b/module2-sql-for-analysis/part1.py @@ -0,0 +1,91 @@ + +# Step 1 - Extract, getting data out of SQLite3 +import sqlite3 +sl_conn = sqlite3.connect('/Users/noahpovis/Desktop/Lambda Clones/DS-Unit-3-Sprint-2-SQL-and-Databases/module1-introduction-to-sql/rpg_db.sqlite3') +sl_curs = sl_conn.cursor() + +# Our goal - copy the charactercreator_character table +get_characters = 'SELECT * FROM charactercreator_character;' +characters = sl_curs.execute(get_characters).fetchall() + +# Step 2 - Transform +# Our goal is to make a schema to define a table that fits this data in Postgres +# Can we check the old schema? +sl_curs.execute('PRAGMA table_info(charactercreator_character);').fetchall() + +## We need to make a create statement for PostgreSQL that captures these types +create_character_table = """ +CREATE TABLE charactercreator_character ( + character_id SERIAL PRIMARY KEY, + name VARCHAR(30), + level INT, + exp INT, + hp INT, + strength INT, + intelligence INT, + dexterity INT, + wisdom INT +); +""" +#!pip install psycopg2-binary +import psycopg2 +dbname = 'njskuvoi' +user = 'njskuvoi' +password = 'V4hoTmOL2IlAb_98ylcdRtErA_ERzFMo' +host = 'ruby.db.elephantsql.com' +pg_conn = psycopg2.connect(dbname=dbname, user=user, + password=password, host=host) +pg_curs = pg_conn.cursor() #works the same as sqlite ! + +# May need to rerun the .connect to refresh +pg_curs = pg_conn.cursor() +pg_curs.execute(create_character_table) +pg_conn.commit() + +# We can query tables if we want to check +# This is a clever optional thing, showing postgresql internals +show_tables = """ +SELECT + * +FROM + pg_catalog.pg_tables +WHERE + schemaname != 'pg_catalog' +AND schemaname != 'information_schema'; +""" +pg_curs.execute(show_tables) +pg_curs.fetchall() + +example_insert = """ +INSERT INTO charactercreator_character +(name, level, exp, hp, strength, intelligence, dexterity, wisdom) +VALUES """ + str(characters[0][1:]) + ";" + +print(example_insert) # Not running, just inspecting + +# If we ran that, we'd insert the first character +# But we want them all - loops! +for character in characters: + insert_character = """ + INSERT INTO charactercreator_character + (name, level, exp, hp, strength, intelligence, dexterity, wisdom) + VALUES """ + str(character[1:]) + ";" + pg_curs.execute(insert_character) + + # PostgreSQL cursor needs to fetch in separate step, unlike SQLite +pg_curs.execute('SELECT * FROM charactercreator_character LIMIT 5;') +pg_curs.fetchall() + +# It inserted, and we can query from our open cursor (because it did the insert) +# But other connections and cursors don't know about it yet - we didn't commit! +pg_conn.commit() + +pg_curs.execute('SELECT * FROM charactercreator_character;') +pg_characters = pg_curs.fetchall() + +# We could do more spot checks, but let's loop and check them all +for character, pg_character in zip(characters, pg_characters): + assert character == pg_character + + + diff --git a/module2-sql-for-analysis/rpg_db.sqlite3 b/module2-sql-for-analysis/rpg_db.sqlite3 new file mode 100644 index 00000000..e69de29b diff --git a/module2-sql-for-analysis/titanic.sqlite3 b/module2-sql-for-analysis/titanic.sqlite3 new file mode 100644 index 00000000..002c532b Binary files /dev/null and b/module2-sql-for-analysis/titanic.sqlite3 differ