Add Feature/readthedocs

README.md → README.rst

@@ -1,13 +1,25 @@
-# eGo dataprocessing
-(geo)data processing, database setup, data validation
+.. image:: https://readthedocs.org/projects/data-processing/badge/?version=feature-readthedocs
+    :target: http://data-processing.readthedocs.io/en/latest/?badge=feature-readthedocs
+    :alt: Documentation Status
+
+.. image:: https://openegoproject.files.wordpress.com/2017/02/open_ego_logo_breit.png?w=400
+
 
-# Copyleft
+===================
+eGo dataprocessing
+===================
 
+(geo)data processing, database setup, data validation
+
+=========
+Copyleft
+=========
 Code licensed under "GNU Affero General Public License Version 3 (AGPL-3.0)"
 It is a collaborative work with several copyright owner:
 Cite as "eGo dataprocessing © Flensburg University of Applied Sciences, Centre for Sustainable Energy Systems © Europa-Universität Flensburg, Centre for Sustainable Energy Systems © Reiner Lemoine Institut © DLR Institute for Networked Energy Systems"
 
-## Installation
+Installation
+============
 
 Installation of latest release
 
@@ -21,14 +33,17 @@ Installation (of developer version) via pip on linux systems as follows
 sudo pip3 install -e <path-to-data_processing-repo>
 ```
 
-### Troubleshooting
+Troubleshooting
+
+On Windows
+***********
 
-#### On Windows
 1. __Problem__: Installation of required package shapely fails. __Solution__: Install pre-build
  wheel from [here](http://www.lfd.uci.edu/~gohlke/pythonlibs/#shapely)
 
 
- ## Run
+Run
+====
 
  You can run data processing by calling command-line script
 

dataprocessing/eGo_data_processing.py

@@ -4,7 +4,7 @@
 Also see corresponding BPML diagram.
 """
 
-__copyright__ = "Reiner Lemoine Institut gGmbH"
+__copyright__ = "Reiner Lemoine Institut"
 __license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)"
 __url__ = "https://github.com/openego/data_processing/blob/master/LICENSE"
 __author__ = "gplssm, Ludee"
@@ -40,58 +40,54 @@ def data_processing():
                      'python_scripts'))
 
     snippets = [
-	## VACUUM FULL
-#	 'ego_dp_vacuum_full.sql',
-
-	## INPUT DATA CHECK eGoPP
-	'ego_dp_structure_input_verification.sql',          	# Check input tables and versions from eGoPP
-
-	## SUBSTATION
-     	'ego_dp_substation_hvmv.sql',               		# Abstract HVMV Substations of the high voltage level from OSM
-     	'ego_dp_substation_ehv.sql',             		# Abstract EHV Substations of the extra high voltage level from OSM
-     	'ego_dp_substation_otg.sql',           			# Assign osmTGmod-id to HVMV and EHV substations
-     	'ego_dp_substation_hvmv_voronoi.sql',			# HVMV Voronoi cells based on HVMV substations
-    	'ego_dp_substation_ehv_voronoi.sql',			# EHV Voronoi cells based on EHV substations
-
-	## GRIDDISTRICT
-   	'ego_dp_mv_griddistrict.sql',          			# MV Griddistricts from municipalities and Voronoi cells
-	'ego_dp_substation_id_to_generator.sql',		# Assign subst_id to pp lists
-
-	## LOADAREA
-	 'ego_dp_loadarea_landuse.sql', 			# OSM landuse sector
-	 'ego_dp_loadarea_industry_consumer.sql', 		# Industry consumer
-	 'ego_dp_loadarea_loads.sql',            		# Loads from OSM landuse
-	 'ego_dp_loadarea_census.sql',   			# Loads from Census
-	 'ego_dp_loadarea_loadmelt.sql',        		# Melt OSM landuse and Zensus loads
-	 'ego_dp_loadarea_loadcut_griddistrict.sql', 		# Cut Loadarea with MV Griddistrict
-	 'ego_dp_loadarea_loadcut_voronoi.sql', 	    	# Cut Loadarea with MV Voronoi cells
-	 'ego_dp_loadarea_consumption.sql',			# Allocate consumption to Loadareas
-	 'ego_dp_loadarea_peakload.sql',			# Peak loads per Loadarea
-	 'ego_dp_loadarea_griddistrict_results.sql',		# Results for MV Griddistrict
-	 'ego_dp_loadarea_statistic.sql',			# Results and statistics for eGoDP
-
-	## LOWVOLTAGE
-	 'ego_dp_lv_substation.sql',				# MVLV Substation inside Loadarea
-	 'ego_dp_lv_substation_voronoi.sql',			# MVLV Substation Voronoi
-	 'ego_dp_lv_griddistrict.sql',    			# LV Griddistrict
-	 'ego_dp_lv_peakload.sql',				# LV OSM areas
-	 'ego_dp_lv_consumption_peakload.sql',			# LV Consumption and Peakload
-
-	## REA
-  	 'rea/ego_dp_lattice_500m.sql', 			# lattice (point grid) 500m
- 	 'rea/ego_dp_lattice_50m.sql', 				# lattice (point grid) 50m
-	 'rea/ego_dp_rea_wpa_per_mvgd.sql', 			# wind potential area
-
-	 'rea/ego_dp_rea_lattice_per_area_500m.sql', 		# prepare 500m lattice
-	 'rea/ego_dp_rea_lattice_per_area_50m.sql', 		# prepare 50m lattice
-   	 'rea/ego_dp_rea_setup.sql',				# setup table for allocation
-   	 'rea/ego_dp_rea_m1.sql',	                   	# M1 biomass and solar to farmyard
-   	 'rea/ego_dp_rea_m2.sql',	                   	# M2 windfarms
-  	 'rea/ego_dp_rea_m3.sql',	                   	# M3 wind turbines to wpa
- 	 'rea/ego_dp_rea_m4.sql',	                   	# M4 other and rest
-    	 'rea/ego_dp_rea_m5.sql',	                   	# M5 LV to LA
-    	 'rea/ego_dp_rea_results.sql',                  	# results and statistics
-
+    ## INPUT DATA CHECK (eGoPP)
+    'ego_dp_structure_input_verification.sql',          # Input verification (eGoPP)
+
+    ## SUBSTATION
+    'ego_dp_substation_hvmv.sql',                       # Abstract HVMV Substations of the high voltage level from OSM
+    'ego_dp_substation_ehv.sql',                        # Abstract EHV Substations of the extra high voltage level from OSM
+    'ego_dp_substation_otg.sql',                        # Assign osmTGmod-id to HVMV and EHV substations
+    'ego_dp_substation_hvmv_voronoi.sql',               # HVMV Voronoi cells based on HVMV substations
+    'ego_dp_substation_ehv_voronoi.sql',                # EHV Voronoi cells based on EHV substations
+
+    ## GRIDDISTRICT
+    'ego_dp_mv_griddistrict.sql',                       # MV GridDistricts from municipalities and Voronoi cells
+    'ego_dp_substation_id_to_generator.sql',            # Assign Substation ID (subst_id) to Generator (Conventional and Renewable)
+
+    ## LOADAREA
+    'ego_dp_loadarea_landuse.sql',                      # OSM landuse sectors
+    'ego_dp_loadarea_industry_consumer.sql',            # OSM Industry consumer
+    'ego_dp_loadarea_loads.sql',                        # OSM Loads from landuse
+    'ego_dp_loadarea_census.sql',                       # Loads from Census 2011
+    'ego_dp_loadarea_loadmelt.sql',                     # Melt loads from OSM landuse and Census 2011
+    'ego_dp_loadarea_loadcut_griddistrict.sql',         # Cut Loadarea with MV Griddistrict
+    'ego_dp_loadarea_loadcut_voronoi.sql',              # Cut Loadarea with MV Voronoi cells
+    'ego_dp_loadarea_consumption.sql',                  # Allocate consumption to Loadareas
+    'ego_dp_loadarea_peakload.sql',                     # Peak loads per Loadarea
+    'ego_dp_loadarea_griddistrict_results.sql',         # Results for MV Griddistrict
+    'ego_dp_loadarea_statistic.sql',                    # Results and statistics for eGoDP data
+
+    ## LOWVOLTAGE
+    'ego_dp_lv_substation.sql',                         # MVLV Substation (ONT)
+    'ego_dp_lv_substation_voronoi.sql',                 # MVLV Substation Voronoi
+    'ego_dp_lv_loadcut.sql'                             # LV Loadcut
+    'ego_dp_lv_griddistrict.sql',                       # LV Griddistrict
+    'ego_dp_lv_consumption_peakload.sql',               # LV Consumption and Peakload
+
+    ## REA
+    'rea/ego_dp_lattice_500m.sql',                      # Lattice (point grid) 500m
+    'rea/ego_dp_lattice_50m.sql',                       # Lattice (point grid) 50m
+    'rea/ego_dp_rea_wpa_per_mvgd.sql',                  # Wind potential area (WPA)
+    'rea/ego_dp_rea_lattice_per_area_500m.sql',         # Prepare 500m lattice
+    'rea/ego_dp_rea_lattice_per_area_50m.sql',          # Prepare 50m lattice
+    'rea/ego_dp_rea_setup.sql',                         # Setup tables for REA
+    'rea/ego_dp_rea_m1.sql',                            # M1 biomass and solar to OSM agricultural
+    'rea/ego_dp_rea_m2.sql',                            # M2 wind farms
+    'rea/ego_dp_rea_m3.sql',                            # M3 wind turbines to WPA
+    'rea/ego_dp_rea_m4.sql',                            # M4 other and rest
+    'rea/ego_dp_rea_m5.sql',                            # M5 LV to Loadarea
+    'rea/ego_dp_rea_results.sql',                       # Results and statistics
+
 	## POWERFLOW
 	'ego_dp_powerflow_assignment_otgid.sql',		# assign otg_id to pp lists
 	'ego_dp_powerflow_assignment_unid.sql',			# create a unified_id over all pp (res and conv) 
@@ -100,6 +96,7 @@ def data_processing():
 	'ego_dp_powerflow_hv_setup.sql',			# Set schema/tables for EHV/HV powerflow calculations up
 	'ego_dp_powerflow_osmtgmod_to_pypsa.sql',		# Include data from osmTGmod into EHV/HV powerflow schema
 	'ego_dp_powerflow_electrical_neighbour.sql',		# Create border crossing lines and buses in neighbouring countries
+	'ego_dp_powerflow_fix_ehv_subnetworks.sql',		# Fix topological errors in eHV grid
 	'ego_dp_powerflow_grid_future_scenarios.sql',		# Copy grid to future scenarios
 	'ego_dp_powerflow_assignment_generator.sql',  		# Assign generators to corresponding substation (SQ, NEP2035, eGo100)
 	'ego_dp_powerflow_assignment_load.sql',        		# Assign loads to their corresponding substation (SQ, NEP2035, eGo100)

dataprocessing/sql_snippets/ego_dp_loadarea_census.sql

@@ -1,11 +1,12 @@
 /*
-census 2011 population per ha 
-Identify population in osm loads
-
-__copyright__ 	= "Reiner Lemoine Institut"
-__license__ 	= "GNU Affero General Public License Version 3 (AGPL-3.0)"
-__url__ 	= "https://github.com/openego/data_processing/blob/master/LICENSE"
-__author__ 	= "Ludee"
+Loads from Census 2011
+Include Census 2011 population per ha.
+Identify population in OSM loads.
+
+__copyright__   = "Reiner Lemoine Institut"
+__license__     = "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__url__         = "https://github.com/openego/data_processing/blob/master/LICENSE"
+__author__      = "Ludee"
 */
 
 
@@ -62,7 +63,7 @@ DELETE FROM	model_draft.ego_demand_la_zensus AS lp
 	WHERE	lp.inside_la IS TRUE;
 
 -- grant (oeuser)
-ALTER TABLE	model_draft.ego_demand_la_zensus OWNER TO oeuser;	
+ALTER TABLE model_draft.ego_demand_la_zensus OWNER TO oeuser;
 
 -- metadata
 COMMENT ON TABLE model_draft.ego_demand_la_zensus IS '{
@@ -96,51 +97,53 @@ INSERT INTO	model_draft.ego_demand_la_zensus_cluster(geom)
     ORDER BY gid;
 
 -- index gist (geom)
-CREATE INDEX	ego_demand_la_zensus_cluster_geom_idx
-	ON 	model_draft.ego_demand_la_zensus_cluster USING GIST (geom);
-
--- cluster data
-UPDATE 	model_draft.ego_demand_la_zensus_cluster AS t1
-	SET  	zensus_sum = t2.zensus_sum,
-		area_ha = t2.area_ha,
-		geom_buffer = t2.geom_buffer,
-		geom_centroid = t2.geom_centroid,
-		geom_surfacepoint = t2.geom_surfacepoint
-	FROM    (
-		SELECT	cl.cid AS cid,
-			SUM(lp.population) AS zensus_sum,
-			COUNT(lp.geom) AS area_ha,
-			ST_BUFFER(cl.geom, 100) AS geom_buffer,
-			ST_Centroid(cl.geom) AS geom_centroid,
-			ST_PointOnSurface(cl.geom) AS geom_surfacepoint
-		FROM	model_draft.ego_demand_la_zensus AS lp,
-			model_draft.ego_demand_la_zensus_cluster AS cl
-		WHERE  	cl.geom && lp.geom AND
-			ST_CONTAINS(cl.geom,lp.geom)
-		GROUP BY	cl.cid
-		ORDER BY	cl.cid
-		) AS t2
-	WHERE  	t1.cid = t2.cid;
+CREATE INDEX ego_demand_la_zensus_cluster_geom_idx
+    ON model_draft.ego_demand_la_zensus_cluster USING GIST (geom);
 
 -- index gist (geom_centroid)
-CREATE INDEX	ego_demand_la_zensus_cluster_geom_centroid_idx
-	ON	model_draft.ego_demand_la_zensus_cluster USING GIST (geom_centroid);
+CREATE INDEX ego_demand_la_zensus_cluster_geom_centroid_idx
+    ON model_draft.ego_demand_la_zensus_cluster USING GIST (geom_centroid);
 
 -- index gist (geom_surfacepoint)
-CREATE INDEX	ego_demand_la_zensus_cluster_geom_surfacepoint_idx
-	ON	model_draft.ego_demand_la_zensus_cluster USING GIST (geom_surfacepoint);
+CREATE INDEX ego_demand_la_zensus_cluster_geom_surfacepoint_idx
+    ON model_draft.ego_demand_la_zensus_cluster USING GIST (geom_surfacepoint);
 
 -- grant (oeuser)
-ALTER TABLE	model_draft.ego_demand_la_zensus_cluster OWNER TO oeuser;
+ALTER TABLE model_draft.ego_demand_la_zensus_cluster OWNER TO oeuser;
 
 -- metadata
 COMMENT ON TABLE model_draft.ego_demand_la_zensus_cluster IS '{
     "comment": "eGoDP - Temporary table", 
     "version": "v0.3.0",
     "published": "none" }';
 
--- select description
-SELECT obj_description('model_draft.ego_demand_la_zensus_cluster' ::regclass) ::json;
+-- insert cluster
+INSERT INTO model_draft.ego_demand_la_zensus_cluster(geom)
+    SELECT  (ST_DUMP(ST_MULTI(ST_UNION(grid.geom)))).geom ::geometry(Polygon,3035) AS geom
+    FROM    model_draft.ego_demand_la_zensus AS grid;
+
+-- cluster data
+UPDATE model_draft.ego_demand_la_zensus_cluster AS t1
+    SET zensus_sum = t2.zensus_sum,
+        area_ha = t2.area_ha,
+        geom_buffer = t2.geom_buffer,
+        geom_centroid = t2.geom_centroid,
+        geom_surfacepoint = t2.geom_surfacepoint
+    FROM    (
+        SELECT  cl.cid AS cid,
+                SUM(lp.population) AS zensus_sum,
+                COUNT(lp.geom) AS area_ha,
+                ST_BUFFER(cl.geom, 100) AS geom_buffer,
+                ST_Centroid(cl.geom) AS geom_centroid,
+                ST_PointOnSurface(cl.geom) AS geom_surfacepoint
+        FROM    model_draft.ego_demand_la_zensus AS lp,
+                model_draft.ego_demand_la_zensus_cluster AS cl
+        WHERE   cl.geom && lp.geom AND
+                ST_CONTAINS(cl.geom,lp.geom)
+        GROUP BY cl.cid
+        ORDER BY cl.cid
+        ) AS t2
+    WHERE   t1.cid = t2.cid;
 
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_demand_la_zensus_cluster','ego_dp_loadarea_census.sql',' ');
@@ -170,7 +173,7 @@ CREATE MATERIALIZED VIEW         	model_draft.ego_society_zensus_per_la_mview AS
 	FROM	model_draft.ego_demand_la_zensus_cluster;
 
 -- grant (oeuser)
-ALTER TABLE	model_draft.ego_society_zensus_per_la_mview OWNER TO oeuser;
+ALTER TABLE model_draft.ego_society_zensus_per_la_mview OWNER TO oeuser;
 
 -- metadata
 COMMENT ON TABLE model_draft.ego_society_zensus_per_la_mview IS '{