From 0a99e617e91ba2fc56daa13ab6a4ce64b29349d3 Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Tue, 6 Sep 2022 21:24:11 -0400 Subject: [PATCH 01/10] Added IMPORT_MODULES setting, default to disabled. --- settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/settings.py b/settings.py index 305a4dc2..5dcf924b 100644 --- a/settings.py +++ b/settings.py @@ -7,6 +7,7 @@ NETBOX_URL = os.getenv("NETBOX_URL") NETBOX_TOKEN = os.getenv("NETBOX_TOKEN") IGNORE_SSL_ERRORS = (os.getenv("IGNORE_SSL_ERRORS", "False") == "True") +IMPORT_MODULES = (os.getenv("IMPORT_MODULES", 'False') == 'True') # optionally load vendors through a comma separated list as env var VENDORS = list(filter(None, os.getenv("VENDORS", "").split(","))) From 926b0ff268091d78fb2f1ebe38624ad56ad7672f Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Wed, 7 Sep 2022 10:04:33 -0400 Subject: [PATCH 02/10] Update nb-dt-import.py Add create_module_bays and activate in createDeviceTypes() --- nb-dt-import.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/nb-dt-import.py b/nb-dt-import.py index 82f6ddd7..9da4b6d7 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -297,6 +297,42 @@ def createDeviceBays(devicebays, deviceType, nb): print(e.error) +def create_module_bays(module_bays, device_type, nb): + '''Create module bays. + + Args: + module_bays: parsed YAML module_bays section. + device_type: the device type instance from netbox. + nb: Netbox API instance + ''' + all_module_bays = { + str(item): item for item in nb.dcim.module_bay_templates.filter( + devicetype_id=device_type + ) + } + need_module_bays = [] + for module_bay in module_bays: + try: + dbGet = all_module_bays[module_bay["name"]] + print(f'Module Bay Template Exists: {dbGet.name} - ' + + f'{dbGet.device_type.id} - {dbGet.id}') + except KeyError: + module_bay['device_type'] = device_type + need_module_bays.append(module_bay) + + if not need_module_bays: + return + + try: + module_bay_res = nb.dcim.module_bay_templates.create(need_module_bays) + for module_bay in module_bay_res: + print(f'Module Bay Created: {module_bay.name} - ' + + f'{module_bay.device_type.id} - {module_bay.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) + + def createPowerOutlets(poweroutlets, deviceType, nb): all_poweroutlets = {str(item): item for item in nb.dcim.power_outlet_templates.filter(devicetype_id=deviceType)} need_poweroutlets = [] @@ -375,6 +411,9 @@ def createDeviceTypes(deviceTypes, nb): if "device-bays" in deviceType: createDeviceBays(deviceType["device-bays"], dt.id, nb) + if settings.IMPORT_MODULES and 'module-bays' in deviceType: + create_module_bays(deviceType['module-bays'], + dt.id, nb) def main(): From 44b8970854fb7039848c8fe0e7358bdd3593aea2 Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Wed, 7 Sep 2022 21:37:23 -0400 Subject: [PATCH 03/10] Modules are now imported as well. --- nb-dt-import.py | 337 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 335 insertions(+), 2 deletions(-) diff --git a/nb-dt-import.py b/nb-dt-import.py index 9da4b6d7..33cf89b9 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -55,6 +55,33 @@ def getFiles(vendors=None): files.extend(glob.glob(base_path + f'[!Testing]*/*.{extension}')) return files, discoveredVendors +def get_files_modules(vendors=None): + + files = [] + discoveredVendors = [] + base_path = './repo/module-types/' + if vendors: + for r, d, f in os.walk(base_path): + for folder in d: + for vendor in vendors: + if vendor.lower() == folder.lower(): + discoveredVendors.append({'name': folder, + 'slug': slugFormat(folder)}) + for extension in YAML_EXTENSIONS: + files.extend( + glob.glob( + base_path + folder + f'/*.{extension}' + ) + ) + else: + for r, d, f in os.walk(base_path): + for folder in d: + if folder.lower() != "Testing": + discoveredVendors.append({'name': folder, + 'slug': slugFormat(folder)}) + for extension in YAML_EXTENSIONS: + files.extend(glob.glob(base_path + f'[!Testing]*/*.{extension}')) + return files, discoveredVendors def readYAMl(files, **kwargs): slugs = kwargs.get('slugs', None) @@ -80,6 +107,29 @@ def readYAMl(files, **kwargs): manufacturers.append(manufacturer) return deviceTypes +def read_yaml_modules(files, **kwargs): + slugs = kwargs.get('slugs', None) + module_types = [] + manufacturers = [] + for file in files: + with open(file, 'r') as stream: + try: + data = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + continue + manufacturer = data['manufacturer'] + data['manufacturer'] = {} + data['manufacturer']['name'] = manufacturer + data['manufacturer']['slug'] = slugFormat(manufacturer) + + if slugs and data['slug'] not in slugs: + print(f"Skipping {data['model']}") + continue + + module_types.append(data) + manufacturers.append(manufacturer) + return module_types def createManufacturers(vendors, nb): all_manufacturers = {str(item): item for item in nb.dcim.manufacturers.all()} @@ -129,7 +179,30 @@ def createInterfaces(interfaces, deviceType, nb): except pynetbox.RequestError as e: print(e.error) +def create_interfaces_modules(interfaces, module_type, nb): + all_interfaces = {str(item): item for item in nb.dcim.interface_templates.filter(moduletype_id=module_type)} + need_interfaces = [] + for interface in interfaces: + try: + if_res = all_interfaces[interface["name"]] + print(f'Interface Template Exists: {if_res.name} - {if_res.type}' + + f' - {if_res.module_type.id} - {if_res.id}') + except KeyError: + interface['module_type'] = module_type + need_interfaces.append(interface) + + if not need_interfaces: + return + try: + ifSuccess = nb.dcim.interface_templates.create(need_interfaces) + for intf in ifSuccess: + print(f'Interface Template Created: {intf.name} - ' + + f'{intf.type} - {intf.module_type.id} - ' + + f'{intf.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createConsolePorts(consoleports, deviceType, nb): all_consoleports = {str(item): item for item in nb.dcim.console_port_templates.filter(devicetype_id=deviceType)} @@ -156,6 +229,30 @@ def createConsolePorts(consoleports, deviceType, nb): except pynetbox.RequestError as e: print(e.error) +def create_module_console_ports(consoleports, module_type, nb): + + all_consoleports = {str(item): item for item in nb.dcim.console_port_templates.filter(moduletype_id=module_type)} + need_consoleports = [] + for consoleport in consoleports: + try: + cpGet = all_consoleports[consoleport["name"]] + print(f'Console Port Template Exists: {cpGet.name} - ' + + f'{cpGet.type} - {cpGet.module_type.id} - {cpGet.id}') + except KeyError: + consoleport['module_type'] = module_type + need_consoleports.append(consoleport) + + if not need_consoleports: + return + + try: + cpSuccess = nb.dcim.console_port_templates.create(need_consoleports) + for port in cpSuccess: + print(f'Console Port Created: {port.name} - {port.type} - ' + + f'{port.module_type.id} - {port.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(f"Error creating module console port: {e.error}") def createPowerPorts(powerports, deviceType, nb): all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(devicetype_id=deviceType)} @@ -164,7 +261,7 @@ def createPowerPorts(powerports, deviceType, nb): try: ppGet = all_power_ports[powerport["name"]] print(f'Power Port Template Exists: {ppGet.name} - ' - + f'{ppGet.type} - {ppGet.device_type.id} - {ppGet.id}') + + f'{ppGet.type} - {ppGet.device_type.id} - {ppGet.id}') except KeyError: powerport['device_type'] = deviceType need_power_ports.append(powerport) @@ -182,6 +279,29 @@ def createPowerPorts(powerports, deviceType, nb): except pynetbox.RequestError as e: print(e.error) +def create_module_power_ports(powerports, module_type, nb): + all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(moduletype_id=module_type)} + need_power_ports = [] + for powerport in powerports: + try: + ppGet = all_power_ports[powerport["name"]] + print(f'Power Port Template Exists: {ppGet.name} - ' + + f'{ppGet.type} - {ppGet.module_type.id} - {ppGet.id}') + except KeyError: + powerport['module_type'] = module_type + need_power_ports.append(powerport) + + if not need_power_ports: + return + + try: + ppSuccess = nb.dcim.power_port_templates.create(need_power_ports) + for pp in ppSuccess: + print(f'Power port template created: {pp.name} - ' + + f'{pp.type} - {pp.module_type.id} - {pp.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createConsoleServerPorts(consoleserverports, deviceType, nb): all_consoleserverports = {str(item): item for item in nb.dcim.console_server_port_templates.filter(devicetype_id=deviceType)} @@ -210,6 +330,32 @@ def createConsoleServerPorts(consoleserverports, deviceType, nb): except pynetbox.RequestError as e: print(e.error) +def create_module_console_server_ports(consoleserverports, module_type, nb): + all_consoleserverports = {str(item): item for item in nb.dcim.console_server_port_templates.filter(moduletype_id=module_type)} + need_consoleserverports = [] + for csport in consoleserverports: + try: + cspGet = all_consoleserverports[csport["name"]] + print(f'Console Server Port Template Exists: {cspGet.name} - ' + + f'{cspGet.type} - {cspGet.module_type.id} - ' + + f'{cspGet.id}') + except KeyError: + csport['module_type'] = module_type + need_consoleserverports.append(csport) + + if not need_consoleserverports: + return + + try: + cspSuccess = nb.dcim.console_server_port_templates.create( + need_consoleserverports) + for csp in cspSuccess: + print(f'Console Server Port Created: {csp.name} - ' + + f'{csp.type} - {csp.module_type.id} - ' + + f'{csp.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createFrontPorts(frontports, deviceType, nb): all_frontports = {str(item): item for item in nb.dcim.front_port_templates.filter(devicetype_id=deviceType)} @@ -245,6 +391,39 @@ def createFrontPorts(frontports, deviceType, nb): except pynetbox.RequestError as e: print(e.error) +def create_module_front_ports(frontports, module_type, nb): + all_frontports = {str(item): item for item in nb.dcim.front_port_templates.filter(moduletype_id=module_type)} + need_frontports = [] + for frontport in frontports: + try: + fpGet = all_frontports[frontport["name"]] + print(f'Front Port Template Exists: {fpGet.name} - ' + + f'{fpGet.type} - {fpGet.module_type.id} - {fpGet.id}') + except KeyError: + frontport['module_type'] = module_type + need_frontports.append(frontport) + + if not need_frontports: + return + + all_rearports = {str(item): item for item in nb.dcim.rear_port_templates.filter(moduletype_id=module_type)} + for port in need_frontports: + try: + rpGet = all_rearports[port["rear_port"]] + port['rear_port'] = rpGet.id + except KeyError: + print(f'Could not find Rear Port for Front Port: {port["name"]} - ' + + f'{port["type"]} - {module_type}') + + try: + fpSuccess = nb.dcim.front_port_templates.create(need_frontports) + for fp in fpSuccess: + print(f'Front Port Created: {fp.name} - ' + + f'{fp.type} - {fp.module_type.id} - ' + + f'{fp.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createRearPorts(rearports, deviceType, nb): all_rearports = {str(item): item for item in nb.dcim.rear_port_templates.filter(devicetype_id=deviceType)} @@ -271,6 +450,30 @@ def createRearPorts(rearports, deviceType, nb): except pynetbox.RequestError as e: print(e.error) +def create_module_rear_ports(rearports, module_type, nb): + all_rearports = {str(item): item for item in nb.dcim.rear_port_templates.filter(moduletype_id=module_type)} + need_rearports = [] + for rearport in rearports: + try: + rpGet = all_rearports[rearport["name"]] + print(f'Rear Port Template Exists: {rpGet.name} - {rpGet.type}' + + f' - {rpGet.module_type.id} - {rpGet.id}') + except KeyError: + rearport['module_type'] = module_type + need_rearports.append(rearport) + + if not need_rearports: + return + + try: + rpSuccess = nb.dcim.rear_port_templates.create( + need_rearports) + for rp in rpSuccess: + print(f'Rear Port Created: {rp.name} - {rp.type}' + + f' - {rp.module_type.id} - {rp.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createDeviceBays(devicebays, deviceType, nb): all_devicebays = {str(item): item for item in nb.dcim.device_bay_templates.filter(devicetype_id=deviceType)} @@ -367,6 +570,73 @@ def createPowerOutlets(poweroutlets, deviceType, nb): except pynetbox.RequestError as e: print(e.error) +def create_module_power_outlets(poweroutlets, module_type, nb): + all_poweroutlets = {str(item): item for item in nb.dcim.power_outlet_templates.filter(moduletype_id=module_type)} + need_poweroutlets = [] + for poweroutlet in poweroutlets: + try: + poGet = all_poweroutlets[poweroutlet["name"]] + print(f'Power Outlet Template Exists: {poGet.name} - {poGet.type}' + + f' - {poGet.module_type.id} - {poGet.id}') + except KeyError: + poweroutlet["module_type"] = module_type + need_poweroutlets.append(poweroutlet) + + if not need_poweroutlets: + return + + all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(moduletype_id=module_type)} + for outlet in need_poweroutlets: + try: + ppGet = all_power_ports[outlet["power_port"]] + outlet['power_port'] = ppGet.id + except KeyError: + pass + + try: + poSuccess = nb.dcim.power_outlet_templates.create( + need_poweroutlets) + for po in poSuccess: + print(f'Power Outlet Created: {po.name} - ' + + f'{po.type} - {po.module_type.id} - ' + + f'{po.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) + +def create_module_power_outlets(poweroutlets, deviceType, nb): + all_poweroutlets = {str(item): item for item in nb.dcim.power_outlet_templates.filter(devicetype_id=deviceType)} + need_poweroutlets = [] + for poweroutlet in poweroutlets: + try: + poGet = all_poweroutlets[poweroutlet["name"]] + print(f'Power Outlet Template Exists: {poGet.name} - ' + + f'{poGet.type} - {poGet.device_type.id} - {poGet.id}') + except KeyError: + poweroutlet["device_type"] = deviceType + need_poweroutlets.append(poweroutlet) + + if not need_poweroutlets: + return + + all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(devicetype_id=deviceType)} + for outlet in need_poweroutlets: + try: + ppGet = all_power_ports[outlet["power_port"]] + outlet['power_port'] = ppGet.id + except KeyError: + pass + + try: + poSuccess = nb.dcim.power_outlet_templates.create( + need_poweroutlets) + for po in poSuccess: + print(f'Power Outlet Created: {po.name} - ' + + f'{po.type} - {po.device_type.id} - ' + + f'{po.id}') + counter.update({'updated': 1}) + except pynetbox.RequestError as e: + print(e.error) def createDeviceTypes(deviceTypes, nb): all_device_types = {str(item): item for item in nb.dcim.device_types.all()} @@ -415,6 +685,54 @@ def createDeviceTypes(deviceTypes, nb): create_module_bays(deviceType['module-bays'], dt.id, nb) +def create_module_types(module_types, nb): + + all_module_types = {} + for curr_nb_mt in nb.dcim.module_types.all(): + if curr_nb_mt.manufacturer.slug not in all_module_types: + all_module_types[curr_nb_mt.manufacturer.slug] = {} + + all_module_types[curr_nb_mt.manufacturer.slug][curr_nb_mt.model] = curr_nb_mt + + + for curr_mt in module_types: + try: + module_type_res = all_module_types[curr_mt['manufacturer']['slug']][curr_mt["model"]] + print(f'Module Type Exists: {module_type_res.manufacturer.name} - ' + + f'{module_type_res.model} - {module_type_res.id}') + except KeyError: + try: + module_type_res = nb.dcim.module_types.create(curr_mt) + counter.update({'added': 1}) + print(f'Module Type Created: {module_type_res.manufacturer.name} - ' + + f'{module_type_res.model} - {module_type_res.id}') + except pynetbox.RequestError as exce: + print(f"Error '{exce.error}' creating module type: " + + f"{curr_mt}") + + #module_type_res = all_module_types[curr_mt['manufacturer']['slug']][curr_mt["model"]] + + if "interfaces" in curr_mt: + create_interfaces_modules(curr_mt["interfaces"], + module_type_res.id, nb) + if "power-ports" in curr_mt: + create_module_power_ports(curr_mt["power-ports"], + module_type_res.id, nb) + if "console-ports" in curr_mt: + create_module_console_ports(curr_mt["console-ports"], + module_type_res.id, nb) + if "power-outlets" in curr_mt: # No current entries to test + create_module_power_outlets(curr_mt["power-outlets"], + module_type_res.id, nb) + if "console-server-ports" in curr_mt: # No current entries to test + create_module_console_server_ports(curr_mt["console-server-ports"], + module_type_res.id, nb) + if "rear-ports" in curr_mt: + create_module_rear_ports(curr_mt["rear-ports"], + module_type_res.id, nb) + if "front-ports" in curr_mt: + create_module_front_ports(curr_mt["front-ports"], + module_type_res.id, nb) def main(): @@ -473,7 +791,22 @@ def main(): deviceTypes = readYAMl(files, slugs=args.slugs) print(str(len(deviceTypes)) + " Device-Types Found") createManufacturers(vendors, nb) - createDeviceTypes(deviceTypes, nb) + #createDeviceTypes(deviceTypes, nb) + + if settings.IMPORT_MODULES: + if not args.vendors: + print("No Vendors Specified, Gathering All Module-Types") + files, vendors = get_files_modules() + else: + print("Vendor Specified, Gathering All Matching Module-Types") + files, vendors = get_files_modules(args.vendors) + + + print(str(len(vendors)) + " Module Vendors Found") + module_types = read_yaml_modules(files, slugs=args.slugs) + print(str(len(module_types)) + " Module-Types Found") + createManufacturers(vendors, nb) + create_module_types(module_types, nb) print('---') print('Script took {} to run'.format(datetime.now() - startTime)) From 397d25e5cae1e053ba6e83605b3918e96c23f3b8 Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Wed, 7 Sep 2022 21:38:26 -0400 Subject: [PATCH 04/10] Reactivate createDeviceTypes which was disabled for testing. --- nb-dt-import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb-dt-import.py b/nb-dt-import.py index 33cf89b9..6f045eda 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -791,7 +791,7 @@ def main(): deviceTypes = readYAMl(files, slugs=args.slugs) print(str(len(deviceTypes)) + " Device-Types Found") createManufacturers(vendors, nb) - #createDeviceTypes(deviceTypes, nb) + createDeviceTypes(deviceTypes, nb) if settings.IMPORT_MODULES: if not args.vendors: From 22106427d071e5384315f4e334a10d0b9bbda004 Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Thu, 8 Sep 2022 09:15:19 -0400 Subject: [PATCH 05/10] Add docstring to get_files_modules --- nb-dt-import.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nb-dt-import.py b/nb-dt-import.py index 6f045eda..384a5255 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -55,8 +55,19 @@ def getFiles(vendors=None): files.extend(glob.glob(base_path + f'[!Testing]*/*.{extension}')) return files, discoveredVendors -def get_files_modules(vendors=None): - +def get_files_modules(vendors: list[str]=None): + '''Get files list for modules. + + Args: + vendors: List of vendors to sync or None to sync all vendors. + + Returns: + A 2-tuple of: + - list of filenames found + - list of vendors found + + ''' + files = [] discoveredVendors = [] base_path = './repo/module-types/' @@ -81,6 +92,7 @@ def get_files_modules(vendors=None): 'slug': slugFormat(folder)}) for extension in YAML_EXTENSIONS: files.extend(glob.glob(base_path + f'[!Testing]*/*.{extension}')) + return files, discoveredVendors def readYAMl(files, **kwargs): @@ -108,6 +120,7 @@ def readYAMl(files, **kwargs): return deviceTypes def read_yaml_modules(files, **kwargs): + slugs = kwargs.get('slugs', None) module_types = [] manufacturers = [] From 9115a7d310f316af3dc0f83b6e5a078cb0ff3903 Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Thu, 8 Sep 2022 09:37:47 -0400 Subject: [PATCH 06/10] Removed duplicate create_module_power_outlets --- nb-dt-import.py | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/nb-dt-import.py b/nb-dt-import.py index 384a5255..2a9d84f9 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -598,7 +598,8 @@ def create_module_power_outlets(poweroutlets, module_type, nb): if not need_poweroutlets: return - all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(moduletype_id=module_type)} + all_power_ports = {str(item): item for item in nb.dcim.\ + power_port_templates.filter(moduletype_id=module_type)} for outlet in need_poweroutlets: try: ppGet = all_power_ports[outlet["power_port"]] @@ -617,40 +618,6 @@ def create_module_power_outlets(poweroutlets, module_type, nb): except pynetbox.RequestError as e: print(e.error) -def create_module_power_outlets(poweroutlets, deviceType, nb): - all_poweroutlets = {str(item): item for item in nb.dcim.power_outlet_templates.filter(devicetype_id=deviceType)} - need_poweroutlets = [] - for poweroutlet in poweroutlets: - try: - poGet = all_poweroutlets[poweroutlet["name"]] - print(f'Power Outlet Template Exists: {poGet.name} - ' - + f'{poGet.type} - {poGet.device_type.id} - {poGet.id}') - except KeyError: - poweroutlet["device_type"] = deviceType - need_poweroutlets.append(poweroutlet) - - if not need_poweroutlets: - return - - all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(devicetype_id=deviceType)} - for outlet in need_poweroutlets: - try: - ppGet = all_power_ports[outlet["power_port"]] - outlet['power_port'] = ppGet.id - except KeyError: - pass - - try: - poSuccess = nb.dcim.power_outlet_templates.create( - need_poweroutlets) - for po in poSuccess: - print(f'Power Outlet Created: {po.name} - ' - + f'{po.type} - {po.device_type.id} - ' - + f'{po.id}') - counter.update({'updated': 1}) - except pynetbox.RequestError as e: - print(e.error) - def createDeviceTypes(deviceTypes, nb): all_device_types = {str(item): item for item in nb.dcim.device_types.all()} for deviceType in deviceTypes: From 6cd282acd8d368fa975ab020860a3ce6c40f59f1 Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Thu, 8 Sep 2022 10:50:49 -0400 Subject: [PATCH 07/10] Added docstring to create_module_types --- nb-dt-import.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nb-dt-import.py b/nb-dt-import.py index 2a9d84f9..b4c393d3 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -666,6 +666,15 @@ def createDeviceTypes(deviceTypes, nb): dt.id, nb) def create_module_types(module_types, nb): + '''Create missing module types. + + Args: + module_types: yaml data from repo. + nb: pynetbox API instance + + Returns: + None + ''' all_module_types = {} for curr_nb_mt in nb.dcim.module_types.all(): From 541a357b9f3b11563db4c42991ab465751a2bddf Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Thu, 8 Sep 2022 12:48:24 -0400 Subject: [PATCH 08/10] Removed manual setting for IMPORT_MODULES. Instead automatically detect via version 3.2 or higher. --- nb-dt-import.py | 41 +++++++++++++++++++++++++++++++++++++++-- settings.py | 4 +++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/nb-dt-import.py b/nb-dt-import.py index b4c393d3..7bafdfc3 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -14,6 +14,28 @@ counter = Counter(added=0, updated=0, manufacturer=0) +def determine_features(nb): + '''Automatically determine the netbox features available. + + Currently only checks for existence of module-types. + + Args: + nb: pynetbox API instance + + Returns: + None + + Raises: + None + ''' + + # nb.version should be the version in the form '3.2' + nb_ver = [int(x) for x in nb.version.split('.')] + + # Later than 3.2 + # Might want to check for the module-types entry as well? + if nb_ver[0] > 3 or (nb_ver[0] == 3 and nb_ver[1] >= 2): + settings.NETBOX_FEATURES['modules'] = True def update_package(path: str, branch: str): try: @@ -584,6 +606,19 @@ def createPowerOutlets(poweroutlets, deviceType, nb): print(e.error) def create_module_power_outlets(poweroutlets, module_type, nb): + '''Create missing module power outlets. + + Args: + poweroutlets: YAML power outlet data. + module_type: Netbox module_type instance. + nb: pynetbox API instance. + + Returns: + None + + Raises: + None + ''' all_poweroutlets = {str(item): item for item in nb.dcim.power_outlet_templates.filter(moduletype_id=module_type)} need_poweroutlets = [] for poweroutlet in poweroutlets: @@ -661,7 +696,7 @@ def createDeviceTypes(deviceTypes, nb): if "device-bays" in deviceType: createDeviceBays(deviceType["device-bays"], dt.id, nb) - if settings.IMPORT_MODULES and 'module-bays' in deviceType: + if settings.NETBOX_FEATURES['modules'] and 'module-bays' in deviceType: create_module_bays(deviceType['module-bays'], dt.id, nb) @@ -732,6 +767,8 @@ def main(): nbToken = settings.NETBOX_TOKEN nb = pynetbox.api(nbUrl, token=nbToken) + determine_features(nb) + if settings.IGNORE_SSL_ERRORS: import requests requests.packages.urllib3.disable_warnings() @@ -782,7 +819,7 @@ def main(): createManufacturers(vendors, nb) createDeviceTypes(deviceTypes, nb) - if settings.IMPORT_MODULES: + if settings.NETBOX_FEATURES['modules']: if not args.vendors: print("No Vendors Specified, Gathering All Module-Types") files, vendors = get_files_modules() diff --git a/settings.py b/settings.py index 5dcf924b..bcbb8553 100644 --- a/settings.py +++ b/settings.py @@ -7,7 +7,6 @@ NETBOX_URL = os.getenv("NETBOX_URL") NETBOX_TOKEN = os.getenv("NETBOX_TOKEN") IGNORE_SSL_ERRORS = (os.getenv("IGNORE_SSL_ERRORS", "False") == "True") -IMPORT_MODULES = (os.getenv("IMPORT_MODULES", 'False') == 'True') # optionally load vendors through a comma separated list as env var VENDORS = list(filter(None, os.getenv("VENDORS", "").split(","))) @@ -15,6 +14,9 @@ # optionally load device types through a space separated list as env var SLUGS = os.getenv("SLUGS", "").split() +NETBOX_FEATURES = { + 'modules': False, +} MANDATORY_ENV_VARS = ["REPO_URL", "NETBOX_URL", "NETBOX_TOKEN"] From e733964bc188ca692b0ff2159e4e8d9bd9292227 Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Thu, 8 Sep 2022 14:53:18 -0400 Subject: [PATCH 09/10] Added module counters separate from device-types. --- nb-dt-import.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/nb-dt-import.py b/nb-dt-import.py index 7bafdfc3..9539a3ca 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -12,7 +12,13 @@ import re -counter = Counter(added=0, updated=0, manufacturer=0) +counter = Counter( + added=0, + updated=0, + manufacturer=0, + module_added=0, + module_port_added=0, +) def determine_features(nb): '''Automatically determine the netbox features available. @@ -214,7 +220,7 @@ def createInterfaces(interfaces, deviceType, nb): except pynetbox.RequestError as e: print(e.error) -def create_interfaces_modules(interfaces, module_type, nb): +def create_module_interfaces(interfaces, module_type, nb): all_interfaces = {str(item): item for item in nb.dcim.interface_templates.filter(moduletype_id=module_type)} need_interfaces = [] for interface in interfaces: @@ -235,7 +241,7 @@ def create_interfaces_modules(interfaces, module_type, nb): print(f'Interface Template Created: {intf.name} - ' + f'{intf.type} - {intf.module_type.id} - ' + f'{intf.id}') - counter.update({'updated': 1}) + counter.update({'module_port_added': 1}) except pynetbox.RequestError as e: print(e.error) @@ -285,7 +291,7 @@ def create_module_console_ports(consoleports, module_type, nb): for port in cpSuccess: print(f'Console Port Created: {port.name} - {port.type} - ' + f'{port.module_type.id} - {port.id}') - counter.update({'updated': 1}) + counter.update({'module_port_added': 1}) except pynetbox.RequestError as e: print(f"Error creating module console port: {e.error}") @@ -334,7 +340,7 @@ def create_module_power_ports(powerports, module_type, nb): for pp in ppSuccess: print(f'Power port template created: {pp.name} - ' + f'{pp.type} - {pp.module_type.id} - {pp.id}') - counter.update({'updated': 1}) + counter.update({'module_port_added': 1}) except pynetbox.RequestError as e: print(e.error) @@ -388,7 +394,7 @@ def create_module_console_server_ports(consoleserverports, module_type, nb): print(f'Console Server Port Created: {csp.name} - ' + f'{csp.type} - {csp.module_type.id} - ' + f'{csp.id}') - counter.update({'updated': 1}) + counter.update({'module_port_added': 1}) except pynetbox.RequestError as e: print(e.error) @@ -456,7 +462,7 @@ def create_module_front_ports(frontports, module_type, nb): print(f'Front Port Created: {fp.name} - ' + f'{fp.type} - {fp.module_type.id} - ' + f'{fp.id}') - counter.update({'updated': 1}) + counter.update({'module_port_added': 1}) except pynetbox.RequestError as e: print(e.error) @@ -506,7 +512,7 @@ def create_module_rear_ports(rearports, module_type, nb): for rp in rpSuccess: print(f'Rear Port Created: {rp.name} - {rp.type}' + f' - {rp.module_type.id} - {rp.id}') - counter.update({'updated': 1}) + counter.update({'module_port_added': 1}) except pynetbox.RequestError as e: print(e.error) @@ -649,7 +655,7 @@ def create_module_power_outlets(poweroutlets, module_type, nb): print(f'Power Outlet Created: {po.name} - ' + f'{po.type} - {po.module_type.id} - ' + f'{po.id}') - counter.update({'updated': 1}) + counter.update({'module_port_added': 1}) except pynetbox.RequestError as e: print(e.error) @@ -727,7 +733,7 @@ def create_module_types(module_types, nb): except KeyError: try: module_type_res = nb.dcim.module_types.create(curr_mt) - counter.update({'added': 1}) + counter.update({'module_added': 1}) print(f'Module Type Created: {module_type_res.manufacturer.name} - ' + f'{module_type_res.model} - {module_type_res.id}') except pynetbox.RequestError as exce: @@ -737,8 +743,8 @@ def create_module_types(module_types, nb): #module_type_res = all_module_types[curr_mt['manufacturer']['slug']][curr_mt["model"]] if "interfaces" in curr_mt: - create_interfaces_modules(curr_mt["interfaces"], - module_type_res.id, nb) + create_module_interfaces(curr_mt["interfaces"], + module_type_res.id, nb) if "power-ports" in curr_mt: create_module_power_ports(curr_mt["power-ports"], module_type_res.id, nb) @@ -839,7 +845,9 @@ def main(): print('{} devices created'.format(counter['added'])) print('{} interfaces/ports updated'.format(counter['updated'])) print('{} manufacturers created'.format(counter['manufacturer'])) - + if settings.NETBOX_FEATURES['modules']: + print(f"{counter['module_added']} modules created") + print(f"{counter['module_port_added']} module interface / ports created") if __name__ == "__main__": main() From 27db043f9d0778a4950d8bd1844ecd0456041394 Mon Sep 17 00:00:00 2001 From: David Mc Ken Date: Fri, 9 Sep 2022 11:01:08 -0400 Subject: [PATCH 10/10] Update nb-dt-import.py Removed type hint which is throwing an error on python 3.8.10 --- nb-dt-import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb-dt-import.py b/nb-dt-import.py index 9539a3ca..85ff7f77 100755 --- a/nb-dt-import.py +++ b/nb-dt-import.py @@ -83,7 +83,7 @@ def getFiles(vendors=None): files.extend(glob.glob(base_path + f'[!Testing]*/*.{extension}')) return files, discoveredVendors -def get_files_modules(vendors: list[str]=None): +def get_files_modules(vendors=None): '''Get files list for modules. Args: