Skip to content

Dataframe for ml #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Dataframe for ml #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e6508b3
<Ginger> retry when access denied init
lakshmie6x Jul 20, 2024
dcde149
<Ginger> adding retryable connection
lakshmie6x Jul 20, 2024
19d4ee2
sw: create ML pipeline
swesingh9 Dec 29, 2024
c4cbacc
sw: cleanup stash
swesingh9 Dec 29, 2024
26feae7
sw: double array changes
swesingh9 Dec 31, 2024
01f6d45
sw: add matrix library
swesingh9 Dec 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions MLPipeline_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
from unittest import TestCase
from e6data_python_connector import Connection

import logging

logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)

class TestDataFrame(TestCase):
def setUp(self) -> None:
self._host = "localhost"
self._catalog = "demogluecatalog"
self._database = "ml_expts"
logging.debug('Trying to connect to engine')
self.e6x_connection = Connection(
host=self._host,
port=9001,
username='[email protected]',
password='Dummy@123',
database=self._database,
catalog=self._catalog
)
logging.debug('Successfully connect to engine.')

def disconnect(self):
self.e6x_connection.close()
self.assertFalse(self.e6x_connection.check_connection())

def tearDown(self) -> None:
self.disconnect()

def test_mlpipeline(self):
self._mlPipeline = self.createMLPipeline()
self._mlPipeline.train_linear_model("select * from ml_expts.boston_housing")
self._mlPipeline.train_linear_model("select * from ml_expts.boston_housing")

self._mlPipeline.execute()

57 changes: 57 additions & 0 deletions MatrixTest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
from unittest import TestCase
from e6data_python_connector import Connection
import numpy as np

import logging

from e6data_python_connector.e6data_grpc import Matrix

logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)

class MatrixTest(TestCase):
def setUp(self) -> None:
self._host = "localhost"
self._catalog = "demogluecatalog"
self._database = "ml_expts"
logging.debug('Trying to connect to engine')
self.e6x_connection = Connection(
host=self._host,
port=9020,
username='[email protected]',
password='Dummy@123',
database=self._database,
catalog=self._catalog
)
self.matrix_ops = Matrix(self.e6x_connection)
logging.debug('Successfully connect to engine.')

def disconnect(self):
self.e6x_connection.close()
self.assertFalse(self.e6x_connection.check_connection())

def tearDown(self) -> None:
self.disconnect()

def test_matrixCompute(self):
matrix1 = [[1, 2], [3, 4]]
matrix2 = [[5, 6], [7, 8]]

print("Matrix1: ", matrix1)
print("Matrix2: ", matrix2)

# Matrix Multiplication
print("Matrix Multiplication:")
self.matrix_ops.matmul(matrix1, matrix2)

# Matrix Addition
print("Matrix Addition:")
self.matrix_ops.add(matrix1, matrix2)

# Matrix Transpose
print("Matrix Transpose:")
self.matrix_ops.transpose(matrix1)

# Matrix Inverse
print("Matrix Inverse:")
self.matrix_ops.inverse(matrix1)
138 changes: 0 additions & 138 deletions e6data_python_connector/date_time_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,138 +1,6 @@
from typing import Final
import pytz

TIME_ZONES = {
"+12:00": ["Pacific/Kwajalein", "Pacific/Wallis", "Pacific/Funafuti", "Pacific/Nauru", "Kwajalein", "Pacific/Wake",
"Pacific/Norfolk", "Pacific/Tarawa", "Asia/Kamchatka", "Etc/GMT-12", "Asia/Anadyr", "Pacific/Majuro", ],
"+14:00": ["Pacific/Kiritimati", "Etc/GMT-14", ],
"UTC": ["UTC"],
"-01:00": ["Etc/GMT+1", "Atlantic/Cape_Verde", "Atlantic/Azores", "America/Scoresbysund", ],
"-03:00": ["Chile/Continental", "America/Miquelon", "America/Argentina/Catamarca", "America/Argentina/Cordoba",
"America/Araguaina", "America/Argentina/Salta", "Etc/GMT+3", "America/Montevideo", "Brazil/East",
"America/Argentina/Mendoza", "America/Argentina/Rio_Gallegos", "America/Catamarca", "America/Godthab",
"America/Cordoba", "America/Sao_Paulo", "America/Argentina/Jujuy", "America/Cayenne", "America/Recife",
"America/Buenos_Aires", "America/Paramaribo", "America/Mendoza", "America/Santarem", "America/Asuncion",
"America/Maceio", "Atlantic/Stanley", "Antarctica/Rothera", "America/Argentina/San_Luis",
"America/Santiago", "America/Argentina/Ushuaia", "Antarctica/Palmer", "America/Punta_Arenas",
"America/Fortaleza", "America/Argentina/La_Rioja", "America/Belem", "America/Jujuy", "America/Bahia",
"America/Argentina/San_Juan", "America/Argentina/ComodRivadavia", "America/Argentina/Tucuman",
"America/Nuuk", "America/Rosario", "America/Argentina/Buenos_Aires", ],
"-09:00": ["Etc/GMT+9", "US/Alaska", "America/Juneau", "America/Metlakatla", "Pacific/Gambier", "America/Yakutat",
"America/Sitka", "SystemV/YST9", "America/Anchorage", "America/Nome", "SystemV/YST9YDT", ],
"-07:00": ["Canada/Yukon", "Etc/GMT+7", "US/Arizona", "Mexico/BajaSur", "America/Dawson_Creek", "America/Denver",
"America/Yellowknife", "America/Inuvik", "America/Mazatlan", "SystemV/MST7", "America/Boise",
"America/Dawson", "MST7MDT", "America/Chihuahua", "America/Ojinaga", "US/Mountain", "America/Creston",
"America/Edmonton", "America/Hermosillo", "Canada/Mountain", "America/Cambridge_Bay", "Navajo",
"America/Phoenix", "America/Whitehorse", "SystemV/MST7MDT", "America/Fort_Nelson", "America/Shiprock", ],
"-05:00": ["America/Panama", "America/Indiana/Petersburg", "America/Eirunepe", "America/Grand_Turk", "Cuba",
"Etc/GMT+5", "Pacific/Easter", "America/Fort_Wayne", "America/Havana", "America/Porto_Acre",
"US/Michigan", "America/Louisville", "America/Guayaquil", "America/Indiana/Vevay",
"America/Indiana/Vincennes", "America/Indianapolis", "America/Iqaluit", "America/Kentucky/Louisville",
"EST5EDT", "America/Nassau", "America/Jamaica", "America/Atikokan", "America/Kentucky/Monticello",
"America/Coral_Harbour", "America/Cayman", "Chile/EasterIsland", "America/Indiana/Indianapolis",
"America/Thunder_Bay", "America/Indiana/Marengo", "America/Bogota", "SystemV/EST5", "US/Eastern",
"Canada/Eastern", "America/Port-au-Prince", "America/Nipigon", "Brazil/Acre", "US/East-Indiana",
"America/Cancun", "America/Lima", "America/Rio_Branco", "America/Detroit", "Jamaica",
"America/Pangnirtung", "America/Montreal", "America/Indiana/Winamac", "America/New_York",
"America/Toronto", "SystemV/EST5EDT", ],
"+03:00": ["Asia/Aden", "Africa/Nairobi", "Europe/Istanbul", "Etc/GMT-3", "Indian/Comoro", "Antarctica/Syowa",
"Africa/Mogadishu", "Africa/Asmera", "Asia/Istanbul", "Europe/Moscow", "Africa/Djibouti",
"Europe/Simferopol", "Africa/Asmara", "Asia/Baghdad", "Africa/Dar_es_Salaam", "Africa/Addis_Ababa",
"Asia/Riyadh", "Asia/Kuwait", "Europe/Kirov", "Africa/Kampala", "Europe/Minsk", "Asia/Qatar",
"Asia/Bahrain", "Indian/Antananarivo", "Indian/Mayotte", "Europe/Volgograd", "Turkey", "W-SU", ],
"+01:00": ["Europe/Brussels", "Europe/Warsaw", "CET", "Etc/GMT-1", "Europe/Luxembourg", "Africa/Tunis",
"Europe/Malta", "Europe/Busingen", "Africa/Malabo", "Europe/Skopje", "Europe/Sarajevo", "Africa/Lagos",
"Europe/Rome", "Africa/Algiers", "Europe/Zurich", "Europe/Gibraltar", "Europe/Vaduz", "Europe/Ljubljana",
"Europe/Berlin", "Europe/Stockholm", "Europe/Budapest", "Europe/Zagreb", "Europe/Paris",
"Africa/Ndjamena", "Africa/Ceuta", "Europe/Prague", "Europe/Copenhagen", "Europe/Vienna",
"Europe/Tirane", "MET", "Europe/Amsterdam", "Africa/Libreville", "Africa/El_Aaiun", "Europe/San_Marino",
"Africa/Douala", "Africa/Brazzaville", "Africa/Porto-Novo", "Poland", "Europe/Andorra", "Europe/Oslo",
"Europe/Podgorica", "Africa/Casablanca", "Africa/Luanda", "Atlantic/Jan_Mayen", "Africa/Kinshasa",
"Europe/Madrid", "Africa/Bangui", "Europe/Belgrade", "Africa/Niamey", "Europe/Bratislava",
"Arctic/Longyearbyen", "Europe/Vatican", "Europe/Monaco", ],
"+04:30": ["Asia/Kabul", ],
"+07:00": ["Asia/Pontianak", "Etc/GMT-7", "Asia/Phnom_Penh", "Asia/Novosibirsk", "Antarctica/Davis", "Asia/Tomsk",
"Asia/Jakarta", "Asia/Barnaul", "Indian/Christmas", "Asia/Ho_Chi_Minh", "Asia/Hovd", "Asia/Bangkok",
"Asia/Vientiane", "Asia/Novokuznetsk", "Asia/Krasnoyarsk", "Asia/Saigon", ],
"+05:45": ["Asia/Kathmandu", "Asia/Katmandu", ],
"+05:00": ["Asia/Aqtau", "Etc/GMT-5", "Asia/Samarkand", "Asia/Karachi", "Asia/Yekaterinburg", "Asia/Dushanbe",
"Indian/Maldives", "Asia/Oral", "Asia/Tashkent", "Antarctica/Mawson", "Asia/Qyzylorda", "Asia/Aqtobe",
"Asia/Ashkhabad", "Asia/Ashgabat", "Asia/Atyrau", "Indian/Kerguelen", ],
"+06:30": ["Asia/Yangon", "Asia/Rangoon", "Indian/Cocos", ],
"+10:00": ["Pacific/Yap", "Pacific/Port_Moresby", "Pacific/Chuuk", "Australia/Queensland", "Pacific/Guam",
"Pacific/Truk", "Asia/Vladivostok", "Pacific/Saipan", "Antarctica/DumontDUrville", "Australia/Brisbane",
"Etc/GMT-10", "Asia/Ust-Nera", "Australia/Lindeman", ],
"+09:00": ["Etc/GMT-9", "Pacific/Palau", "Asia/Chita", "Asia/Dili", "Asia/Jayapura", "Asia/Yakutsk",
"Asia/Pyongyang", "ROK", "Asia/Seoul", "Asia/Khandyga", "Japan", "Asia/Tokyo", ],
"Z": ["UTC", "Europe/London", "GMT", "Etc/GMT-0", "Europe/Jersey", "Atlantic/St_Helena", "Europe/Guernsey",
"Europe/Isle_of_Man", "Etc/GMT+0", "Africa/Banjul", "Etc/GMT", "Africa/Freetown", "GB-Eire", "Africa/Bamako",
"GB", "Africa/Conakry", "Portugal", "Universal", "Africa/Sao_Tome", "Africa/Nouakchott", "Antarctica/Troll",
"UTC", "Etc/Universal", "Atlantic/Faeroe", "Africa/Abidjan", "Eire", "Africa/Accra", "Atlantic/Faroe",
"Etc/UCT", "GMT0", "Europe/Dublin", "Zulu", "Africa/Ouagadougou", "Atlantic/Reykjavik", "Atlantic/Madeira",
"Etc/Zulu", "Iceland", "Europe/Lisbon", "Atlantic/Canary", "Africa/Lome", "Greenwich", "Europe/Belfast",
"Etc/GMT0", "America/Danmarkshavn", "Africa/Dakar", "Africa/Bissau", "WET", "Etc/Greenwich",
"Africa/Timbuktu", "UCT", "Africa/Monrovia", "Etc/UTC", ],
"-11:00": ["Pacific/Pago_Pago", "Pacific/Samoa", "Pacific/Niue", "US/Samoa", "Etc/GMT+11", "Pacific/Midway", ],
"-06:00": ["America/El_Salvador", "America/Guatemala", "America/Belize", "America/Managua", "America/Chicago",
"America/Tegucigalpa", "Etc/GMT+6", "America/Regina", "Mexico/General", "America/Rankin_Inlet",
"US/Central", "Pacific/Galapagos", "America/Rainy_River", "America/Swift_Current", "America/Costa_Rica",
"America/Indiana/Knox", "America/North_Dakota/Beulah", "America/Monterrey", "SystemV/CST6",
"America/North_Dakota/Center", "America/Indiana/Tell_City", "America/Mexico_City", "America/Matamoros",
"CST6CDT", "America/Knox_IN", "America/Menominee", "America/Resolute", "Canada/Central",
"America/Bahia_Banderas", "US/Indiana-Starke", "SystemV/CST6CDT", "America/Merida",
"Canada/Saskatchewan", "America/North_Dakota/New_Salem", "America/Winnipeg", ],
"-03:30": ["America/St_Johns", "Canada/Newfoundland", ],
"+13:45": ["NZ-CHAT", "Pacific/Chatham", ],
"+10:30": ["Australia/Yancowinna", "Australia/Adelaide", "Australia/Broken_Hill", "Australia/South", ],
"+11:00": ["Australia/Hobart", "Australia/Tasmania", "Australia/ACT", "Pacific/Ponape", "Pacific/Bougainville",
"Australia/Victoria", "Antarctica/Macquarie", "Antarctica/Casey", "Australia/Canberra",
"Australia/Currie", "Australia/Lord_Howe", "Australia/NSW", "Pacific/Pohnpei", "Pacific/Efate",
"Asia/Magadan", "Pacific/Kosrae", "Australia/Sydney", "Australia/LHI", "Asia/Sakhalin", "Pacific/Noumea",
"Etc/GMT-11", "Asia/Srednekolymsk", "Australia/Melbourne", "Pacific/Guadalcanal", ],
"+13:00": ["Pacific/Fakaofo", "Antarctica/McMurdo", "Pacific/Fiji", "Pacific/Enderbury", "NZ", "Pacific/Apia",
"Antarctica/South_Pole", "Pacific/Kanton", "Pacific/Auckland", "Pacific/Tongatapu", "Etc/GMT-13", ],
"-09:30": ["Pacific/Marquesas", ],
"-08:00": ["Etc/GMT+8", "Canada/Pacific", "PST8PDT", "Pacific/Pitcairn", "Mexico/BajaNorte", "America/Tijuana",
"SystemV/PST8", "America/Santa_Isabel", "America/Vancouver", "America/Ensenada", "SystemV/PST8PDT",
"America/Los_Angeles", "US/Pacific", ],
"-02:00": ["Etc/GMT+2", "America/Noronha", "Brazil/DeNoronha", "Atlantic/South_Georgia", ],
"-04:00": ["America/Cuiaba", "America/Marigot", "Canada/Atlantic", "Etc/GMT+4", "America/Manaus",
"America/St_Thomas", "America/Anguilla", "America/Barbados", "America/Curacao", "America/Guyana",
"America/Martinique", "America/Puerto_Rico", "America/Port_of_Spain", "SystemV/AST4",
"America/Kralendijk", "America/Antigua", "America/Moncton", "America/St_Vincent", "America/Dominica",
"Atlantic/Bermuda", "Brazil/West", "America/Aruba", "America/Halifax", "America/La_Paz",
"America/Blanc-Sablon", "America/Santo_Domingo", "America/Glace_Bay", "America/St_Barthelemy",
"America/St_Lucia", "America/Montserrat", "America/Lower_Princes", "America/Thule", "America/Tortola",
"America/Porto_Velho", "America/Campo_Grande", "America/Goose_Bay", "America/Virgin",
"America/Boa_Vista", "America/Grenada", "America/St_Kitts", "America/Caracas", "America/Guadeloupe",
"SystemV/AST4ADT", ],
"+02:00": ["Africa/Cairo", "Africa/Mbabane", "Etc/GMT-2", "Europe/Zaporozhye", "Libya", "Africa/Kigali",
"Africa/Tripoli", "Israel", "Europe/Kaliningrad", "Africa/Windhoek", "Europe/Bucharest",
"Europe/Mariehamn", "Africa/Lubumbashi", "Europe/Tiraspol", "Europe/Chisinau", "Europe/Helsinki",
"Asia/Beirut", "Asia/Tel_Aviv", "Europe/Sofia", "Africa/Gaborone", "Asia/Gaza", "Europe/Riga",
"Africa/Maputo", "Asia/Damascus", "Europe/Uzhgorod", "Asia/Jerusalem", "Africa/Bujumbura", "Europe/Kiev",
"Europe/Vilnius", "Africa/Maseru", "Africa/Blantyre", "Africa/Lusaka", "Africa/Harare", "Europe/Tallinn",
"Africa/Khartoum", "Africa/Johannesburg", "Africa/Juba", "Asia/Nicosia", "Asia/Famagusta", "EET",
"Asia/Hebron", "Egypt", "Asia/Amman", "Europe/Nicosia", "Europe/Athens", ],
"+03:30": ["Iran", "Asia/Tehran", ],
"+04:00": ["Asia/Yerevan", "Etc/GMT-4", "Asia/Dubai", "Indian/Reunion", "Indian/Mauritius", "Europe/Saratov",
"Europe/Samara", "Indian/Mahe", "Asia/Baku", "Asia/Muscat", "Europe/Astrakhan", "Asia/Tbilisi",
"Europe/Ulyanovsk", ],
"+06:00": ["Asia/Kashgar", "Etc/GMT-6", "Asia/Almaty", "Asia/Dacca", "Asia/Omsk", "Asia/Dhaka", "Indian/Chagos",
"Asia/Qostanay", "Asia/Bishkek", "Antarctica/Vostok", "Asia/Urumqi", "Asia/Thimbu", "Asia/Thimphu", ],
"+08:45": ["Australia/Eucla", ],
"+08:00": ["Asia/Kuching", "Asia/Chungking", "Etc/GMT-8", "Australia/Perth", "Asia/Macao", "Asia/Macau",
"Asia/Choibalsan", "Asia/Shanghai", "Asia/Ulan_Bator", "Asia/Chongqing", "Asia/Ulaanbaatar",
"Asia/Taipei", "Asia/Manila", "PRC", "Asia/Ujung_Pandang", "Asia/Harbin", "Singapore", "Asia/Brunei",
"Australia/West", "Asia/Hong_Kong", "Asia/Makassar", "Hongkong", "Asia/Kuala_Lumpur", "Asia/Irkutsk",
"Asia/Singapore", ],
"-12:00": ["Etc/GMT+12", ],
"+05:30": ["Asia/Kolkata", "Asia/Colombo", "Asia/Calcutta", ],
"+09:30": ["Australia/North", "Australia/Darwin", ],
"-10:00": ["Pacific/Honolulu", "Pacific/Rarotonga", "Pacific/Tahiti", "Pacific/Johnston", "America/Atka",
"US/Hawaii", "SystemV/HST10", "America/Adak", "US/Aleutian", "Etc/GMT+10", ],

}

FORMATS = {
"yyyy": "%Y",
Expand Down Expand Up @@ -182,9 +50,3 @@ def get_format(str_format):
if str_format is None or len(str_format) == 0:
return "%Y-%m-%d %H:%M:%S"
return FORMATS[str_format]


def get_zone(zone):
if zone is None or len(zone) == 0:
return "UTC"
return TIME_ZONES[zone][0]
Loading