diff --git a/module1-introduction-to-sql/README.md b/module1-introduction-to-sql/README.md index 40497956..644a6d89 100644 --- a/module1-introduction-to-sql/README.md +++ b/module1-introduction-to-sql/README.md @@ -52,13 +52,70 @@ Use `sqlite3` to load and write queries to explore the data, and answer the following questions: - How many total Characters are there? + SELECT * FROM charactercreator_character + + There are a total of 302 characters. + - How many of each specific subclass? + There are 75 cleric characters. + There are 68 fighter characters. + There are 108 mage characters. + There are only 11 necromancers (a subset of mage characters). + There are 51 thief characters. + - How many total Items? + There are total items in the armory. + - How many of the Items are weapons? How many are not? + 37 items in the armory are weapons, which means 137 items are not weapons. + - How many Items does each character have? (Return first 20 rows) + SELECT * FROM charactercreator_character_inventory + LIMIT 20 + + The first 20 rows of charactercreator_character_inventory tells us that: + - character 1 has three items + - character 2 has three items + - character 3 has two items + - character 4 has four items + - character 5 has four items + - character 6 has one item + - character 7 has at least 3 items. + - How many Weapons does each character have? (Return first 20 rows) + SELECT armory_weapon.item_ptr_id, character_id, COUNT(item_ptr_id) + FROM armory_weapon + INNER JOIN charactercreator_character_inventory ON charactercreator_character_inventory.item_id = armory_weapon.item_ptr_id + GROUP BY charactercreator_character_inventory.character_id + ORDER BY character_id ASC; + + Of the first 20 characters, the following characters have the following number weapons: + - character 5 has 2 weapons + - character 7 has 1 weapon + - character 11 has 1 weapon + - charactrer 20 has 1 weapon. + - On average, how many Items does each Character have? + SELECT AVG(items_per_char) + FROM( + SELECT COUNT(charactercreator_character_inventory.item_id) as items_per_char + FROM charactercreator_character_inventory + GROUP BY charactercreator_character_inventory.character_id) + + Looks like each character has an average of 2.97 items. + + - On average, how many Weapons does each character have? + SELECT AVG(weapons) + FROM + (SELECT COUNT(item_ptr_id) as weapons + FROM( + SELECT * + FROM charactercreator_character_inventory + LEFT JOIN armory_weapon ON charactercreator_character_inventory.item_id = armory_weapon.item_ptr_id) + GROUP BY character_id) + + It appears that the average number of weapons across all characters (including ones that do not have any weapons in their inventory) is 0.67. You do not need all the tables - in particular, the `account_*`, `auth_*`, `django_*`, and `socialaccount_*` tables are for the application and do not have diff --git a/module1-introduction-to-sql/buddymove_holidayiq.py b/module1-introduction-to-sql/buddymove_holidayiq.py new file mode 100644 index 00000000..f22a55ae --- /dev/null +++ b/module1-introduction-to-sql/buddymove_holidayiq.py @@ -0,0 +1,39 @@ +import os +import sqlite3 +import pandas as pd + +# Load the data (use `pandas`) from the provided file `buddymove_holidayiq.csv` + +df = pd.read_csv('https://raw.githubusercontent.com/KristineYW/DS-Unit-3-Sprint-2-SQL-and-Databases/master/module1-introduction-to-sql/buddymove_holidayiq.csv') + +print(df.head()) +print(df.shape) +print(df.isnull().sum()) + +# Open a connection to a new (blank) database file `buddymove_holidayiq.sqlite3` + +filepath = os.path.join(os.path.dirname(__file__), "buddymove_holidayiq.sqlite3") + +connection = sqlite3.connect(filepath) + +# Use `df.to_sql` to insert the data into a new table `review` in the SQLite3 database + +df.to_sql("review", connection) + +# Count how many rows you have - it should be 249! + +query = "SELECT COUNT (*) FROM review;" +cursor = connection.cursor() + +result1 = cursor.execute(query).fetchall() +print("NUMBER OF REVIEWERS:", result1) + + +# How many users who reviewed at least 100 `Nature` in the category also +# reviewed at least 100 in the `Shopping` category? + +query2 = "SELECT COUNT(*) FROM review WHERE Nature > 100 AND Shopping > 100" +result2 = cursor.execute(query2).fetchall() +print("NUMBER OF REVIEWERS WHO REVIEWED OVER 100 NATURE AND OVER 100 SHOPPING CATEGORIES:",result2) + + diff --git a/module1-introduction-to-sql/buddymove_holidayiq.sqlite3 b/module1-introduction-to-sql/buddymove_holidayiq.sqlite3 new file mode 100644 index 00000000..5354df30 Binary files /dev/null and b/module1-introduction-to-sql/buddymove_holidayiq.sqlite3 differ diff --git a/module1-introduction-to-sql/chinook.db b/module1-introduction-to-sql/chinook.db new file mode 100755 index 00000000..38a98b39 Binary files /dev/null and b/module1-introduction-to-sql/chinook.db differ diff --git a/module1-introduction-to-sql/chinook_queries.py b/module1-introduction-to-sql/chinook_queries.py new file mode 100644 index 00000000..d6bf06ef --- /dev/null +++ b/module1-introduction-to-sql/chinook_queries.py @@ -0,0 +1,28 @@ +import os +import sqlite3 + +# construct a path to wherever your database exists +#DB_FILEPATH = "chinook.db" +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "chinook.db") + +connection = sqlite3.connect(DB_FILEPATH) +print("CONNECTION:", connection) + +cursor = connection.cursor() +print("CURSOR", cursor) + +query = "SELECT * FROM customers;" + +#result = cursor.execute(query) +#print("RESULT", result) #> returns cursor object w/o results (need to fetch the results) + +result2 = cursor.execute(query).fetchall() +print("RESULT 2", result2) + +for row in result2: + print(type(row), row) + + query2 = "SELECT count(distinct CustomerId) as customer_count FROM customers;" + +result3 = cursor.execute(query2).fetchone() +print("RESULT 3", type(result3), result3) \ No newline at end of file diff --git a/module2-sql-for-analysis/elephant_q.py b/module2-sql-for-analysis/elephant_q.py new file mode 100644 index 00000000..83f9b75f --- /dev/null +++ b/module2-sql-for-analysis/elephant_q.py @@ -0,0 +1,49 @@ +import os +import json +from dotenv import load_dotenv +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() #> loads contents of the .env file into the script's environment +DB_NAME = os.getenv("DB_NAME") +DB_USER = os.getenv("DB_USER") +DB_PASSWORD = os.getenv("DB_PASSWORD") +DB_HOST = os.getenv("DB_HOST") +print(DB_NAME, DB_USER, DB_PASSWORD, DB_HOST) +connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) +print("CONNECTION", connection) +cursor = connection.cursor() +print("CURSOR", cursor) + +# FETCH DATA +# FYI: in sqlite: result = cursor.execute("SELECT * from test_table;").fetchall() +cursor.execute("SELECT * from test_table;") +result = cursor.fetchall() +print(result) + +# INSERT DATA +#insertion_sql = """ +#INSERT INTO test_table (name, data) VALUES +#('A row name', null), +#('Another row, with JSON','{ "a": 1, "b": ["dog", "cat", 42], "c": true }'::JSONB); +#""" +#cursor.execute(insertion_sql) +my_dict = { "a": 1, "b": ["dog", "cat", 42], "c": 'true' } +#insertion_query = "INSERT INTO test_table (name, data) VALUES (%s, %s)" +#cursor.execute(insertion_query, +# ('A rowwwww', 'null') +#) +#cursor.execute(insertion_query, +# ('Another row, with JSONNNNN', json.dumps(my_dict)) +#) +insertion_query = "INSERT INTO test_table (name, data) VALUES %s" +execute_values(cursor, insertion_query, [ + ('A rowwwww', 'null'), + ('Another row, with JSONNNNN', json.dumps(my_dict)), + ('Third row', "3") +]) # data must be in a list of tuples!!!!! + +# ACTUALLY SAVE THE TRANSACTIONS +connection.commit() +cursor.close() +connection.close() \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_arm_weap.py b/module2-sql-for-analysis/insert_rpg_arm_weap.py new file mode 100644 index 00000000..4d356f16 --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_arm_weap.py @@ -0,0 +1,65 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") + +class SqliteService_armory_weapon(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_armory_weapon(self): #fix + return self.cursor.execute("SELECT * FROM armory_weapon;").fetchall() #fix + +class ElephantSQLService_armory_weapon(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + + def create_armory_weapon_table(self): #fix + create_query = """ + DROP TABLE IF EXISTS armory_weapon; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "armory_weapon_pkey" DETAIL: Key (armory_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS armory_weapon ( + item_ptr_id INT, + power INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + + def insert_armory_weapon(self, armory): + """ + Param armory_weapon needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO armory_weapon (item_ptr_id, power) + VALUES %s + """ + execute_values(self.cursor, insertion_query, armory_weapon) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService_armory_weapon() + armory_weapon = sqlite_service.fetch_armory_weapon() + print(type(armory_weapon), len(armory_weapon)) + print(type(armory_weapon[0]), armory_weapon[0]) + # + # LOAD + # + pg_service = ElephantSQLService_armory_weapon() + pg_service.create_armory_weapon_table() + pg_service.insert_armory_weapon(armory_weapon) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_armory.py b/module2-sql-for-analysis/insert_rpg_armory.py new file mode 100644 index 00000000..701bd7e4 --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_armory.py @@ -0,0 +1,67 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") + +class SqliteService_armory(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_armory(self): #fix + return self.cursor.execute("SELECT * FROM armory_item;").fetchall() #fix + +class ElephantSQLService_armory(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + + def create_armory_table(self): #fix + create_query = """ + DROP TABLE IF EXISTS armory; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "armory_pkey" DETAIL: Key (armory_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS armory ( + item_id SERIAL PRIMARY KEY, + name VARCHAR(255), + value INT, + weight INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + + def insert_armory(self, armory): + """ + Param armory needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO armory (item_id, name, value, weight) + VALUES %s + """ + execute_values(self.cursor, insertion_query, armory) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService_armory() + armory = sqlite_service.fetch_armory() + print(type(armory), len(armory)) + print(type(armory[0]), armory[0]) + # + # LOAD + # + pg_service = ElephantSQLService_armory() + pg_service.create_armory_table() + pg_service.insert_armory(armory) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_char_inv.py b/module2-sql-for-analysis/insert_rpg_char_inv.py new file mode 100644 index 00000000..91078f18 --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_char_inv.py @@ -0,0 +1,62 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") +class SqliteService_inventory(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_characters_inventory(self): + return self.cursor.execute("SELECT * FROM charactercreator_character_inventory;").fetchall() +class ElephantSQLService_inventory(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + def create_characters_inventory_table(self): + create_query = """ + DROP TABLE IF EXISTS characters_inventory; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "characters__inventory_pkey" DETAIL: Key (character_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS characters_inventory ( + id SERIAL PRIMARY KEY, + character_id INT, + item_id INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + def insert_characters_inventory(self, characters_inventory): + """ + Param characters_inventory needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO characters_inventory (id, character_id,item_id) + VALUES %s + """ + execute_values(self.cursor, insertion_query, characters_inventory) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService_inventory() + characters_inventory = sqlite_service.fetch_characters_inventory() + print(type(characters_inventory), len(characters_inventory)) + print(type(characters_inventory[0]), characters_inventory[0]) + # + # LOAD + # + pg_service = ElephantSQLService_inventory() + pg_service.create_characters_inventory_table() + pg_service.insert_characters_inventory(characters_inventory) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_character.py b/module2-sql-for-analysis/insert_rpg_character.py new file mode 100644 index 00000000..31f463bf --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_character.py @@ -0,0 +1,68 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") +class SqliteService(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_characters(self): + return self.cursor.execute("SELECT * FROM charactercreator_character;").fetchall() +class ElephantSQLService(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + def create_characters_table(self): + create_query = """ + DROP TABLE IF EXISTS characters; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "characters_pkey" DETAIL: Key (character_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS characters ( + character_id SERIAL PRIMARY KEY, + name VARCHAR(30), + level INT, + exp INT, + hp INT, + strength INT, + intelligence INT, + dexterity INT, + wisdom INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + def insert_characters(self, characters): + """ + Param characters needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO characters (character_id, name, level, exp, hp, strength, intelligence, dexterity, wisdom) + VALUES %s + """ + execute_values(self.cursor, insertion_query, characters) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService() + characters = sqlite_service.fetch_characters() + print(type(characters), len(characters)) + print(type(characters[0]), characters[0]) + # + # LOAD + # + pg_service = ElephantSQLService() + pg_service.create_characters_table() + pg_service.insert_characters(characters) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_cleric.py b/module2-sql-for-analysis/insert_rpg_cleric.py new file mode 100644 index 00000000..6310ed88 --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_cleric.py @@ -0,0 +1,62 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") +class SqliteService_cleric(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_characters_cleric(self): + return self.cursor.execute("SELECT * FROM charactercreator_cleric;").fetchall() +class ElephantSQLService_cleric(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + def create_characters_cleric_table(self): + create_query = """ + DROP TABLE IF EXISTS characters_cleric; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "characters_cleric_pkey" DETAIL: Key (character_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS characters_cleric ( + character_ptr_id INT, + using_shield INT, + mana INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + def insert_characters_cleric(self, characters_cleric): + """ + Param characters_cleric needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO characters_cleric (character_ptr_id, using_shield, mana) + VALUES %s + """ + execute_values(self.cursor, insertion_query, characters_cleric) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService_cleric() + characters_cleric = sqlite_service.fetch_characters_cleric() + print(type(characters_cleric), len(characters_cleric)) + print(type(characters_cleric[0]), characters_cleric[0]) + # + # LOAD + # + pg_service = ElephantSQLService_cleric() + pg_service.create_characters_cleric_table() + pg_service.insert_characters_cleric(characters_cleric) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_fighter.py b/module2-sql-for-analysis/insert_rpg_fighter.py new file mode 100644 index 00000000..222f429c --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_fighter.py @@ -0,0 +1,62 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") +class SqliteService_fighter(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_characters_fighter(self): + return self.cursor.execute("SELECT * FROM charactercreator_fighter;").fetchall() +class ElephantSQLService_fighter(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + def create_characters_fighter_table(self): + create_query = """ + DROP TABLE IF EXISTS characters_fighter; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "characters_fighter_pkey" DETAIL: Key (character_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS characters_fighter ( + character_ptr_id INT, + using_shield INT, + rage INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + def insert_characters_fighter(self, characters_fighter): + """ + Param characters_fighter needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO characters_fighter (character_ptr_id, using_shield, rage) + VALUES %s + """ + execute_values(self.cursor, insertion_query, characters_fighter) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService_fighter() + characters_fighter = sqlite_service.fetch_characters_fighter() + print(type(characters_fighter), len(characters_fighter)) + print(type(characters_fighter[0]), characters_fighter[0]) + # + # LOAD + # + pg_service = ElephantSQLService_fighter() + pg_service.create_characters_fighter_table() + pg_service.insert_characters_fighter(characters_fighter) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_mage.py b/module2-sql-for-analysis/insert_rpg_mage.py new file mode 100644 index 00000000..dbc68d5a --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_mage.py @@ -0,0 +1,62 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") +class SqliteService_mage(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_characters_mage(self): + return self.cursor.execute("SELECT * FROM charactercreator_mage;").fetchall() +class ElephantSQLService_mage(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + def create_characters_mage_table(self): + create_query = """ + DROP TABLE IF EXISTS characters_mage; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "characters_mage_pkey" DETAIL: Key (character_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS characters_mage ( + character_ptr_id INT, + has_pet INT, + mana INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + def insert_characters_mage(self, characters_mage): + """ + Param characters_mage needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO characters_mage (character_ptr_id, has_pet, mana) + VALUES %s + """ + execute_values(self.cursor, insertion_query, characters_mage) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService_mage() + characters_mage = sqlite_service.fetch_characters_mage() + print(type(characters_mage), len(characters_mage)) + print(type(characters_mage[0]), characters_mage[0]) + # + # LOAD + # + pg_service = ElephantSQLService_mage() + pg_service.create_characters_mage_table() + pg_service.insert_characters_mage(characters_mage) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_necromancer.py b/module2-sql-for-analysis/insert_rpg_necromancer.py new file mode 100644 index 00000000..595d0a6e --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_necromancer.py @@ -0,0 +1,61 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") +class SqliteService_necromancer(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_characters_necromancer(self): + return self.cursor.execute("SELECT * FROM charactercreator_necromancer;").fetchall() +class ElephantSQLService_necromancer(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + def create_characters_necromancer_table(self): + create_query = """ + DROP TABLE IF EXISTS characters_necromancer; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "characters_necromancer_pkey" DETAIL: Key (character_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS characters_necromancer ( + mage_ptr_id INT, + talisman_charged INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + def insert_characters_necromancer(self, characters_necromancer): + """ + Param characters_necromancer needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO characters_necromancer (mage_ptr_id, talisman_charged) + VALUES %s + """ + execute_values(self.cursor, insertion_query, characters_necromancer) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService_necromancer() + characters_necromancer = sqlite_service.fetch_characters_necromancer() + print(type(characters_necromancer), len(characters_necromancer)) + print(type(characters_necromancer[0]), characters_necromancer[0]) + # + # LOAD + # + pg_service = ElephantSQLService_necromancer() + pg_service.create_characters_necromancer_table() + pg_service.insert_characters_necromancer(characters_necromancer) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_rpg_thief.py b/module2-sql-for-analysis/insert_rpg_thief.py new file mode 100644 index 00000000..6da7a648 --- /dev/null +++ b/module2-sql-for-analysis/insert_rpg_thief.py @@ -0,0 +1,62 @@ +import os +from dotenv import load_dotenv +import sqlite3 +import psycopg2 +from psycopg2.extras import execute_values + +load_dotenv() # looks inside the .env file for some env vars + +# passes env var values to python var +DB_HOST = os.getenv("DB_HOST", default="OOPS") +DB_NAME = os.getenv("DB_NAME", default="OOPS") +DB_USER = os.getenv("DB_USER", default="OOPS") +DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS") + +# what is the filepath to connect to our sqlite database? +DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") +class SqliteService_thief(): + def __init__(self, db_filepath=DB_FILEPATH): + self.connection = sqlite3.connect(db_filepath) + self.cursor = self.connection.cursor() + def fetch_characters_thief(self): + return self.cursor.execute("SELECT * FROM charactercreator_thief;").fetchall() +class ElephantSQLService_thief(): + def __init__(self): + self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + self.cursor = self.connection.cursor() + def create_characters_thief_table(self): + create_query = """ + DROP TABLE IF EXISTS characters_thief; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "characters_thief_pkey" DETAIL: Key (character_id)=(1) already exists. + CREATE TABLE IF NOT EXISTS characters_thief ( + character_ptr_id INT, + is_sneaking INT, + energy INT + ); + """ + print(create_query) + self.cursor.execute(create_query) + self.connection.commit() + def insert_characters_thief(self, characters_thief): + """ + Param characters_thief needs to be a list of tuples, each representing a row to insert (each should have each column) + """ + insertion_query = """ + INSERT INTO characters_thief (character_ptr_id, is_sneaking, energy) + VALUES %s + """ + execute_values(self.cursor, insertion_query, characters_thief) + self.connection.commit() +if __name__ == "__main__": + # + # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY) + # + sqlite_service = SqliteService_thief() + characters_thief = sqlite_service.fetch_characters_thief() + print(type(characters_thief), len(characters_thief)) + print(type(characters_thief[0]), characters_thief[0]) + # + # LOAD + # + pg_service = ElephantSQLService_thief() + pg_service.create_characters_thief_table() + pg_service.insert_characters_thief(characters_thief) \ No newline at end of file diff --git a/module2-sql-for-analysis/insert_titanic.py b/module2-sql-for-analysis/insert_titanic.py new file mode 100644 index 00000000..0351f3c5 --- /dev/null +++ b/module2-sql-for-analysis/insert_titanic.py @@ -0,0 +1,55 @@ +import os +import json +from dotenv import load_dotenv +import psycopg2 +from psycopg2.extras import execute_values +import pandas as pd + +# Load contents of the .env file into the script's environment +load_dotenv() + +# Add credentials for accessing the elephant +DB_NAME = os.getenv("DB_NAME") +DB_USER = os.getenv("DB_USER") +DB_PASSWORD = os.getenv("DB_PASSWORD") +DB_HOST = os.getenv("DB_HOST") + +# Load the dataframe with pandas +df = pd.read_csv('https://raw.githubusercontent.com/LambdaSchool/DS-Unit-3-Sprint-2-SQL-and-Databases/master/module2-sql-for-analysis/titanic.csv') + +# Create the psychopg2 connection and cursor objections to access the elephant +connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST) + +cursor = connection.cursor() + +# Create the titanic table +create_query = "CREATE TABLE titanic (Survived INT, Class INT, Name VARCHAR(255), Sex CHAR(10), Age FLOAT, Sibling_Spouse INT, Parent_Child INT, Fare FLOAT);" + +# Create placeholder insertion query for the titanic table +# insertion_query = "INSERT INTO titanic2 (Survived, Class, Name, Sex, Age, Sibling_Spouse, Parent_Child, Fare) VALUES %s" +insertion_query = f"INSERT INTO titanic (Survived, Class, Name, Sex, Age, Sibling_Spouse, Parent_Child, Fare)" \ + "VALUES ({Survivor},{Class},{Name},{Sex},{Age},{Sibing_Spouse},{Parent_Child},{Fare})" + + +# Change the format of database into a list of tuples +list_of_tuples = [] +for row in df.iterrows(): + list_of_tuples.append(row) +print(list_of_tuples) + +lines = [] +for i in range(0,len(df)): + result = [] + for j in range(0,8): + result.append(list_of_tuples[i][1][j]) + lines.append(tuple(result)) +print(lines) + +# Use execute_values to insert the list of tuples into the titanic table as rows + +execute_values(cursor, insertion_query, lines) + +# Save the transactions +connection.commit() +cursor.close() +connection.close() diff --git a/module2-sql-for-analysis/sandbox.ipynb b/module2-sql-for-analysis/sandbox.ipynb new file mode 100644 index 00000000..e69de29b diff --git a/module2-sql-for-analysis/titanic_test.py b/module2-sql-for-analysis/titanic_test.py new file mode 100644 index 00000000..6fa25fff --- /dev/null +++ b/module2-sql-for-analysis/titanic_test.py @@ -0,0 +1,13 @@ +import csv +import psycopg2 +conn = psycopg2.connect("host=localhost dbname=postgres user=postgres") +cur = conn.cursor() +with open('titanic.csv', 'r') as f: + reader = csv.reader(f) + next(reader) # Skip the header row. + for row in reader: + cur.execute( + "INSERT INTO users VALUES (%s, %s, %s, %s, %s, %s ,%s, %s)", + row + ) +conn.commit() \ No newline at end of file