Toby_Chen_DSPT6

module1-introduction-to-sql/buddymove_holidayiq.py

@@ -0,0 +1,15 @@
+import sqlite3
+import pandas as pd
+
+connect = sqlite3.connect('buddymove_holidayiq.sqlite3')
+cursor = connect.cursor()
+
+df = pd.read_csv('buddymove_holidayiq.csv')
+
+df.to_sql('buddy', connect)
+
+cursor.execute('SELECT COUNT(*) FROM buddy')
+print('1. ', cursor.fetchall())
+
+cursor.execute('SELECT COUNT(*) FROM buddy WHERE Nature > 99 AND Shopping > 99')
+print('2. ', cursor.fetchall())

module1-introduction-to-sql/rpg_queries.py

@@ -0,0 +1,62 @@
+import sqlite3
+
+connect = sqlite3.connect('rpg_db.sqlite3')
+cursor = connect.cursor()
+
+cursor.execute('SELECT COUNT(character_id) FROM charactercreator_character')
+print("1. ", cursor.fetchall())
+
+cursor.execute('SELECT COUNT(character_ptr_id) FROM charactercreator_thief')
+print("2.Thief: ", cursor.fetchall())
+
+cursor.execute('SELECT COUNT(character_ptr_id) FROM charactercreator_cleric')
+print("Cleric: ", cursor.fetchall())
+
+cursor.execute('SELECT COUNT(character_ptr_id) FROM charactercreator_mage')
+print("Mage: ", cursor.fetchall())
+
+cursor.execute('SELECT COUNT(character_ptr_id) FROM charactercreator_fighter')
+print("Fighter: ", cursor.fetchall())
+
+cursor.execute('SELECT COUNT(mage_ptr_id) FROM charactercreator_necromancer')
+print("Necromancer: ", cursor.fetchall())
+
+cursor.execute('SELECT COUNT(item_id) FROM armory_item')
+num_item = cursor.fetchall()
+print('3. ', num_item)
+
+cursor.execute('SELECT COUNT(item_ptr_id) FROM armory_weapon')
+num_weapon = cursor.fetchall()
+print(
+    '4. Weapons: ',
+    num_weapon,
+    ' Items: ',
+    num_item[0][0] -
+    num_weapon[0][0])
+
+cursor.execute('''SELECT COUNT(item_id)
+FROM charactercreator_character_inventory
+GROUP BY character_id
+LIMIT 20''')
+print('5. ', cursor.fetchall())
+
+cursor.execute('''SELECT COUNT(item_id)
+FROM charactercreator_character_inventory
+WHERE item_id > 137
+GROUP BY character_id
+LIMIT 20''')
+print('6. ', cursor.fetchall())
+
+
+cursor.execute('''SELECT AVG(a.rcount) FROM
+(select count(*) as rcount
+FROM charactercreator_character_inventory cci
+GROUP BY character_id) a''')
+print('7. ', cursor.fetchall())
+
+cursor.execute('''SELECT AVG(a.rcount) FROM
+(select count(*) as rcount
+FROM charactercreator_character_inventory cci
+WHERE item_id > 137
+GROUP BY character_id) a''')
+print('8. ', cursor.fetchall())

module2-sql-for-analysis/Pipfile

@@ -0,0 +1,13 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+python-dotenv = "*"
+psycopg2-binary = "*"
+
+[requires]
+python_version = "3.7"

module2-sql-for-analysis/insert_titanic.py

@@ -0,0 +1,73 @@
+import os
+from dotenv import load_dotenv
+import psycopg2
+from psycopg2.extras import execute_values
+import pandas
+
+load_dotenv() #> loads contents of the .env file into the script's environment
+
+DB_NAME = os.getenv("DB_NAME2")
+DB_USER = os.getenv("DB_USER2")
+DB_PASS = os.getenv("DB_PASS2")
+DB_HOST = os.getenv("DB_HOST2")
+
+CSV_FILEPATH = "titanic.csv"
+
+# CONNECT TO THE PG DATABASE
+
+connection = psycopg2.connect(dbname=DB_NAME, user=DB_USER, password=DB_PASS, host=DB_HOST)
+cursor = connection.cursor()
+
+
+# CREATE A TABLE TO STORE THE PASSENGERS
+#
+# ... optionally renaming some of the columns, adding a primary key, and changing survived to a bool
+
+sql = """
+DROP TABLE IF EXISTS passengers;
+CREATE TABLE IF NOT EXISTS passengers (
+    id SERIAL PRIMARY KEY,
+    survived boolean,
+    pclass int4,
+    full_name text,
+    gender text,
+    age int4,
+    sib_spouse_count int4,
+    parent_child_count int4,
+    fare float8
+);
+"""
+cursor.execute(sql)
+
+#
+# READ PASSENGER DATA FROM THE CSV FILE
+#
+
+df = pandas.read_csv(CSV_FILEPATH)
+print(df.columns.tolist())
+
+# to avoid PG insertion errors related to datatype mismatches (psycopg2.ProgrammingError: can't adapt type 'numpy.int64'),
+# ... we need to convert np.int64 columns to normal integers
+# ... https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.astype.html
+# ... https://stackoverflow.com/questions/34838378/dataframe-values-tolist-datatype
+# ... https://stackoverflow.com/questions/47423930/how-to-convert-pandas-dataframe-columns-to-native-python-data-types
+
+df["Survived"] = df["Survived"].values.astype(bool) # do this before converting to native types, because this actually converts to np.bool
+df = df.astype("object") # converts numpy dtypes to native python dtypes (avoids psycopg2.ProgrammingError: can't adapt type 'numpy.int64')
+
+#
+# INSERT DATA INTO THE PASSENGERS TABLE
+#
+
+# how to convert dataframe to a list of tuples
+list_of_tuples = list(df.to_records(index=False))
+
+insertion_query = f"INSERT INTO passengers (survived, pclass, full_name, gender, age, sib_spouse_count, parent_child_count, fare) VALUES %s"
+execute_values(cursor, insertion_query, list_of_tuples) # third param: data as a list of tuples!
+
+# CLEAN UP
+connection.commit() # actually save the records / run the transaction to insert rows
+print('Titanic Data successfully saved to Postgres!')
+
+cursor.close()
+connection.close()

module2-sql-for-analysis/rpg_db.py

@@ -0,0 +1,52 @@
+import sqlite3
+import os
+import psycopg2
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DB_NAME2 = os.getenv("DB_NAME3")
+DB_USER2 = os.getenv("DB_USER3")
+DB_PASS2 = os.getenv("DB_PASS3")
+DB_HOST2 = os.getenv("DB_HOST3")
+
+conn = psycopg2.connect(dbname=DB_NAME2, 
+                        user=DB_USER2,
+                        password=DB_PASS2, 
+                        host=DB_HOST2)
+
+cursor = conn.cursor()
+
+sl_conn = sqlite3.connect("rpg_db.sqlite3")
+sl_cursor = sl_conn.cursor()
+characters = sl_cursor.execute('SELECT * FROM charactercreator_character LIMIT 10').fetchall()
+print(characters)
+
+create_character_table_query = '''
+CREATE TABLE IF NOT EXISTS rpg_characters (
+    character_id SERIAL PRIMARY KEY,
+	name VARCHAR(30),
+	level INT,
+	exp INT,
+	hp INT,
+	strength INT, 
+	intelligence INT,
+	dexterity INT,
+	wisdom INT
+)
+'''
+
+cursor.execute(create_character_table_query)
+conn.commit()
+
+for character in characters:
+
+    insert_query = f''' INSERT INTO rpg_characters 
+        (character_id, name, level, exp, hp, strength, intelligence, dexterity, wisdom) VALUES
+        {character}
+    '''
+    cursor.execute(insert_query)
+
+conn.commit()
+cursor.close()
+conn.close()

module3-nosql-and-document-oriented-databases/Pipfile

@@ -0,0 +1,13 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+python-dotenv = "*"
+psycopg2-binary = "*"
+
+[requires]
+python_version = "3.7"