Kristine Wang | DS15 | Unit 3: Sprint 2

module1-introduction-to-sql/README.md

@@ -52,13 +52,70 @@ Use `sqlite3` to load and write queries to explore the data, and answer the
 following questions:
 
 - How many total Characters are there?
+    SELECT * FROM charactercreator_character
+
+    There are a total of 302 characters.
+
 - How many of each specific subclass?
+    There are 75 cleric characters.
+    There are 68 fighter characters.
+    There are 108 mage characters.
+    There are only 11 necromancers (a subset of mage characters).
+    There are 51 thief characters.
+
 - How many total Items?
+    There are total items in the armory.
+
 - How many of the Items are weapons? How many are not?
+    37 items in the armory are weapons, which means 137 items are not weapons. 
+
 - How many Items does each character have? (Return first 20 rows)
+    SELECT * FROM charactercreator_character_inventory 
+    LIMIT 20
+
+    The first 20 rows of charactercreator_character_inventory tells us that:
+      - character 1 has three items 
+      - character 2 has three items
+      - character 3 has two items
+      - character 4 has four items
+      - character 5 has four items
+      - character 6 has one item 
+      - character 7 has at least 3 items.
+
 - How many Weapons does each character have? (Return first 20 rows)
+    SELECT armory_weapon.item_ptr_id, character_id, COUNT(item_ptr_id)
+    FROM armory_weapon
+    INNER JOIN charactercreator_character_inventory ON charactercreator_character_inventory.item_id = armory_weapon.item_ptr_id
+    GROUP BY charactercreator_character_inventory.character_id
+    ORDER BY character_id ASC;
+
+    Of the first 20 characters, the following characters have the following number weapons:
+      - character 5 has 2 weapons
+      - character 7 has 1 weapon
+      - character 11 has 1 weapon
+      - charactrer 20 has 1 weapon.
+
 - On average, how many Items does each Character have?
+    SELECT AVG(items_per_char)
+    FROM(
+      SELECT COUNT(charactercreator_character_inventory.item_id) as items_per_char
+      FROM charactercreator_character_inventory
+      GROUP BY charactercreator_character_inventory.character_id)
+
+    Looks like each character has an average of 2.97 items.
+
+
 - On average, how many Weapons does each character have?
+    SELECT AVG(weapons)
+    FROM 
+    (SELECT COUNT(item_ptr_id) as weapons
+    FROM(
+    	SELECT *
+        FROM charactercreator_character_inventory
+        LEFT JOIN armory_weapon ON charactercreator_character_inventory.item_id = armory_weapon.item_ptr_id)
+    GROUP BY character_id)
+
+    It appears that the average number of weapons across all characters (including ones that do not have any weapons in their inventory) is 0.67.
 
 You do not need all the tables - in particular, the `account_*`, `auth_*`,
 `django_*`, and `socialaccount_*` tables are for the application and do not have

module1-introduction-to-sql/buddymove_holidayiq.py

@@ -0,0 +1,39 @@
+import os
+import sqlite3
+import pandas as pd
+
+# Load the data (use `pandas`) from the provided file `buddymove_holidayiq.csv`
+
+df = pd.read_csv('https://raw.githubusercontent.com/KristineYW/DS-Unit-3-Sprint-2-SQL-and-Databases/master/module1-introduction-to-sql/buddymove_holidayiq.csv')
+
+print(df.head())
+print(df.shape)
+print(df.isnull().sum())
+
+# Open a connection to a new (blank) database file `buddymove_holidayiq.sqlite3`
+
+filepath = os.path.join(os.path.dirname(__file__), "buddymove_holidayiq.sqlite3")
+
+connection = sqlite3.connect(filepath)
+
+# Use `df.to_sql` to insert the data into a new table `review` in the SQLite3 database
+
+df.to_sql("review", connection)
+
+# Count how many rows you have - it should be 249!
+
+query = "SELECT COUNT (*) FROM review;"
+cursor = connection.cursor()
+
+result1 = cursor.execute(query).fetchall()
+print("NUMBER OF REVIEWERS:", result1)
+
+
+#   How many users who reviewed at least 100 `Nature` in the category also
+#   reviewed at least 100 in the `Shopping` category?
+
+query2 = "SELECT COUNT(*) FROM review WHERE Nature > 100 AND Shopping > 100"
+result2 = cursor.execute(query2).fetchall()
+print("NUMBER OF REVIEWERS WHO REVIEWED OVER 100 NATURE AND OVER 100 SHOPPING CATEGORIES:",result2)
+
+

module1-introduction-to-sql/chinook_queries.py

@@ -0,0 +1,28 @@
+import os
+import sqlite3
+
+# construct a path to wherever your database exists
+#DB_FILEPATH = "chinook.db"
+DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "chinook.db")
+
+connection = sqlite3.connect(DB_FILEPATH)
+print("CONNECTION:", connection)
+
+cursor = connection.cursor()
+print("CURSOR", cursor)
+
+query = "SELECT * FROM customers;"
+
+#result = cursor.execute(query)
+#print("RESULT", result) #> returns cursor object w/o results (need to fetch the results)
+
+result2 = cursor.execute(query).fetchall()
+print("RESULT 2", result2)
+
+for row in result2:
+    print(type(row), row)
+
+    query2 = "SELECT count(distinct CustomerId) as customer_count FROM customers;"
+
+result3 = cursor.execute(query2).fetchone()
+print("RESULT 3", type(result3), result3)

module2-sql-for-analysis/elephant_q.py

@@ -0,0 +1,49 @@
+import os
+import json
+from dotenv import load_dotenv
+import psycopg2
+from psycopg2.extras import execute_values
+
+load_dotenv() #> loads contents of the .env file into the script's environment
+DB_NAME = os.getenv("DB_NAME")
+DB_USER = os.getenv("DB_USER")
+DB_PASSWORD = os.getenv("DB_PASSWORD")
+DB_HOST = os.getenv("DB_HOST")
+print(DB_NAME, DB_USER, DB_PASSWORD, DB_HOST)
+connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST)
+print("CONNECTION", connection)
+cursor = connection.cursor()
+print("CURSOR", cursor)
+
+# FETCH DATA
+# FYI: in sqlite: result = cursor.execute("SELECT * from test_table;").fetchall()
+cursor.execute("SELECT * from test_table;")
+result = cursor.fetchall()
+print(result)
+
+# INSERT DATA
+#insertion_sql = """
+#INSERT INTO test_table (name, data) VALUES
+#('A row name', null),
+#('Another row, with JSON','{ "a": 1, "b": ["dog", "cat", 42], "c": true }'::JSONB);
+#"""
+#cursor.execute(insertion_sql)
+my_dict = { "a": 1, "b": ["dog", "cat", 42], "c": 'true' }
+#insertion_query = "INSERT INTO test_table (name, data) VALUES (%s, %s)"
+#cursor.execute(insertion_query,
+#  ('A rowwwww', 'null')
+#)
+#cursor.execute(insertion_query,
+#  ('Another row, with JSONNNNN', json.dumps(my_dict))
+#)
+insertion_query = "INSERT INTO test_table (name, data) VALUES %s"
+execute_values(cursor, insertion_query, [
+  ('A rowwwww', 'null'),
+  ('Another row, with JSONNNNN', json.dumps(my_dict)),
+  ('Third row', "3")
+]) # data must be in a list of tuples!!!!!
+
+# ACTUALLY SAVE THE TRANSACTIONS
+connection.commit()
+cursor.close()
+connection.close()

module2-sql-for-analysis/insert_rpg_arm_weap.py

@@ -0,0 +1,65 @@
+import os
+from dotenv import load_dotenv
+import sqlite3
+import psycopg2
+from psycopg2.extras import execute_values
+
+load_dotenv() # looks inside the .env file for some env vars
+
+# passes env var values to python var
+DB_HOST = os.getenv("DB_HOST", default="OOPS")
+DB_NAME = os.getenv("DB_NAME", default="OOPS")
+DB_USER = os.getenv("DB_USER", default="OOPS")
+DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS")
+
+# what is the filepath to connect to our sqlite database?
+DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") 
+
+class SqliteService_armory_weapon():
+    def __init__(self, db_filepath=DB_FILEPATH):
+        self.connection = sqlite3.connect(db_filepath)
+        self.cursor = self.connection.cursor()
+    def fetch_armory_weapon(self): #fix
+        return self.cursor.execute("SELECT * FROM armory_weapon;").fetchall() #fix
+
+class ElephantSQLService_armory_weapon():
+    def __init__(self):
+        self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST)
+        self.cursor = self.connection.cursor()
+
+    def create_armory_weapon_table(self): #fix
+        create_query = """
+        DROP TABLE IF EXISTS armory_weapon; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "armory_weapon_pkey" DETAIL:  Key (armory_id)=(1) already exists.
+        CREATE TABLE IF NOT EXISTS armory_weapon (
+            item_ptr_id INT,
+            power INT
+        );
+        """
+        print(create_query)
+        self.cursor.execute(create_query)
+        self.connection.commit()
+
+    def insert_armory_weapon(self, armory):
+        """
+        Param armory_weapon needs to be a list of tuples, each representing a row to insert (each should have each column)
+        """
+        insertion_query = """
+            INSERT INTO armory_weapon (item_ptr_id, power)
+            VALUES %s
+        """
+        execute_values(self.cursor, insertion_query, armory_weapon)
+        self.connection.commit()
+if __name__ == "__main__":
+    #
+    # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY)
+    #
+    sqlite_service = SqliteService_armory_weapon()
+    armory_weapon = sqlite_service.fetch_armory_weapon()
+    print(type(armory_weapon), len(armory_weapon))
+    print(type(armory_weapon[0]), armory_weapon[0])
+    #
+    # LOAD
+    #
+    pg_service = ElephantSQLService_armory_weapon()
+    pg_service.create_armory_weapon_table()
+    pg_service.insert_armory_weapon(armory_weapon)

module2-sql-for-analysis/insert_rpg_armory.py

@@ -0,0 +1,67 @@
+import os
+from dotenv import load_dotenv
+import sqlite3
+import psycopg2
+from psycopg2.extras import execute_values
+
+load_dotenv() # looks inside the .env file for some env vars
+
+# passes env var values to python var
+DB_HOST = os.getenv("DB_HOST", default="OOPS")
+DB_NAME = os.getenv("DB_NAME", default="OOPS")
+DB_USER = os.getenv("DB_USER", default="OOPS")
+DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS")
+
+# what is the filepath to connect to our sqlite database?
+DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3") 
+
+class SqliteService_armory():
+    def __init__(self, db_filepath=DB_FILEPATH):
+        self.connection = sqlite3.connect(db_filepath)
+        self.cursor = self.connection.cursor()
+    def fetch_armory(self): #fix
+        return self.cursor.execute("SELECT * FROM armory_item;").fetchall() #fix
+
+class ElephantSQLService_armory():
+    def __init__(self):
+        self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST)
+        self.cursor = self.connection.cursor()
+
+    def create_armory_table(self): #fix
+        create_query = """
+        DROP TABLE IF EXISTS armory; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "armory_pkey" DETAIL:  Key (armory_id)=(1) already exists.
+        CREATE TABLE IF NOT EXISTS armory (
+            item_id SERIAL PRIMARY KEY,
+            name VARCHAR(255),
+            value INT,
+            weight INT
+        );
+        """
+        print(create_query)
+        self.cursor.execute(create_query)
+        self.connection.commit()
+
+    def insert_armory(self, armory):
+        """
+        Param armory needs to be a list of tuples, each representing a row to insert (each should have each column)
+        """
+        insertion_query = """
+            INSERT INTO armory (item_id, name, value, weight)
+            VALUES %s
+        """
+        execute_values(self.cursor, insertion_query, armory)
+        self.connection.commit()
+if __name__ == "__main__":
+    #
+    # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY)
+    #
+    sqlite_service = SqliteService_armory()
+    armory = sqlite_service.fetch_armory()
+    print(type(armory), len(armory))
+    print(type(armory[0]), armory[0])
+    #
+    # LOAD
+    #
+    pg_service = ElephantSQLService_armory()
+    pg_service.create_armory_table()
+    pg_service.insert_armory(armory)

module2-sql-for-analysis/insert_rpg_char_inv.py

@@ -0,0 +1,62 @@
+import os
+from dotenv import load_dotenv
+import sqlite3
+import psycopg2
+from psycopg2.extras import execute_values
+
+load_dotenv() # looks inside the .env file for some env vars
+
+# passes env var values to python var
+DB_HOST = os.getenv("DB_HOST", default="OOPS")
+DB_NAME = os.getenv("DB_NAME", default="OOPS")
+DB_USER = os.getenv("DB_USER", default="OOPS")
+DB_PASSWORD = os.getenv("DB_PASSWORD", default="OOPS")
+
+# what is the filepath to connect to our sqlite database?
+DB_FILEPATH = os.path.join(os.path.dirname(__file__), "..", "module1-introduction-to-sql", "rpg_db.sqlite3")
+class SqliteService_inventory():
+    def __init__(self, db_filepath=DB_FILEPATH):
+        self.connection = sqlite3.connect(db_filepath)
+        self.cursor = self.connection.cursor()
+    def fetch_characters_inventory(self):
+        return self.cursor.execute("SELECT * FROM charactercreator_character_inventory;").fetchall()
+class ElephantSQLService_inventory():
+    def __init__(self):
+        self.connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST)
+        self.cursor = self.connection.cursor()
+    def create_characters_inventory_table(self):
+        create_query = """
+        DROP TABLE IF EXISTS characters_inventory; -- allows this to be run idempotently, avoids psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "characters__inventory_pkey" DETAIL:  Key (character_id)=(1) already exists.
+        CREATE TABLE IF NOT EXISTS characters_inventory (
+            id SERIAL PRIMARY KEY,
+            character_id INT,
+            item_id INT
+        );
+        """
+        print(create_query)
+        self.cursor.execute(create_query)
+        self.connection.commit()
+    def insert_characters_inventory(self, characters_inventory):
+        """
+        Param characters_inventory needs to be a list of tuples, each representing a row to insert (each should have each column)
+        """
+        insertion_query = """
+            INSERT INTO characters_inventory (id, character_id,item_id)
+            VALUES %s
+        """
+        execute_values(self.cursor, insertion_query, characters_inventory)
+        self.connection.commit()
+if __name__ == "__main__":
+    #
+    # EXTRACT (AND MAYBE TRANSFORM IF NECESSARY)
+    #
+    sqlite_service = SqliteService_inventory()
+    characters_inventory = sqlite_service.fetch_characters_inventory()
+    print(type(characters_inventory), len(characters_inventory))
+    print(type(characters_inventory[0]), characters_inventory[0])
+    #
+    # LOAD
+    #
+    pg_service = ElephantSQLService_inventory()
+    pg_service.create_characters_inventory_table()
+    pg_service.insert_characters_inventory(characters_inventory)