merge Dev into feature/readthedocs

README.md

@@ -5,7 +5,7 @@
 
 Code licensed under "GNU Affero General Public License Version 3 (AGPL-3.0)"
 It is a collaborative work with several copyright owner:
-Cite as "eGo dataprocessing � Flensburg University of Applied Sciences, Centre for Sustainable Energy Systems � Europa-Universit�t Flensburg, Centre for Sustainable Energy Systems � Reiner Lemoine Institut � DLR Institute for Networked Energy Systems"
+Cite as "eGo dataprocessing © Flensburg University of Applied Sciences, Centre for Sustainable Energy Systems © Europa-Universität Flensburg, Centre for Sustainable Energy Systems © Reiner Lemoine Institut © DLR Institute for Networked Energy Systems"
 
 ## Installation
 

dataprocessing/eGo_data_processing.py

@@ -100,6 +100,7 @@ def data_processing():
 	'ego_dp_powerflow_hv_setup.sql',			# Set schema/tables for EHV/HV powerflow calculations up
 	'ego_dp_powerflow_osmtgmod_to_pypsa.sql',		# Include data from osmTGmod into EHV/HV powerflow schema
 	'ego_dp_powerflow_electrical_neighbour.sql',		# Create border crossing lines and buses in neighbouring countries
+	'ego_dp_powerflow_fix_ehv_subnetworks.sql',		# Fix topological errors in eHV grid
 	'ego_dp_powerflow_grid_future_scenarios.sql',		# Copy grid to future scenarios
 	'ego_dp_powerflow_assignment_generator.sql',  		# Assign generators to corresponding substation (SQ, NEP2035, eGo100)
 	'ego_dp_powerflow_assignment_load.sql',        		# Assign loads to their corresponding substation (SQ, NEP2035, eGo100)

dataprocessing/sql_snippets/ego_dp_loadarea_census.sql

@@ -10,7 +10,7 @@ __author__ 	= "Ludee"
 
 
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
-SELECT ego_scenario_log('v0.3.0','input','society','destatis_zensus_population_per_ha_mview','ego_dp_loadarea_census.sql',' ');
+SELECT ego_scenario_log('v0.3.0','input','model_draft','destatis_zensus_population_per_ha_invg_mview','ego_dp_loadarea_census.sql',' ');
 
 -- zensus load
 DROP TABLE IF EXISTS  	model_draft.ego_demand_la_zensus CASCADE;
@@ -25,12 +25,13 @@ CREATE TABLE         	model_draft.ego_demand_la_zensus (
 
 -- insert zensus loads
 INSERT INTO	model_draft.ego_demand_la_zensus (gid,population,inside_la,geom_point,geom)
-	SELECT	zensus.gid ::integer,
-		zensus.population ::integer,
+	SELECT	gid ::integer,
+		population ::integer,
 		'FALSE' ::boolean AS inside_la,
-		zensus.geom_point ::geometry(Point,3035),
-		zensus.geom ::geometry(Polygon,3035)
-	FROM	society.destatis_zensus_population_per_ha_mview AS zensus;
+		geom_point ::geometry(Point,3035),
+		geom ::geometry(Polygon,3035)
+	FROM	model_draft.destatis_zensus_population_per_ha_invg_mview
+    ORDER BY gid;
 
 -- index gist (geom_point)
 CREATE INDEX  	ego_demand_la_zensus_geom_point_idx
@@ -65,32 +66,9 @@ ALTER TABLE	model_draft.ego_demand_la_zensus OWNER TO oeuser;
 
 -- metadata
 COMMENT ON TABLE model_draft.ego_demand_la_zensus IS '{
-    "Name": "ego zensus loads",
-    "Source":   [{
-	"Name": "open_eGo",
-	"URL": "https://github.com/openego/data_processing"}],
-    "Reference date": "2016",
-    "Date of collection": "02.09.2016",
-    "Original file": ["ego_grid_hvmv_substation"],
-    "Spatial": [{
-	"Resolution": "",
-	"Extend": "Germany" }],
-    "Description": ["osm laods"],
-    "Column":[
-        {"Name": "id", "Description": "Unique identifier", "Unit": " " },
-        {"Name": "area_ha", "Description": "Area", "Unit": "ha" },
-	{"Name": "geom", "Description": "Geometry", "Unit": " " } ],
-    "Changes":	[
-        {"Name": "Ludee", "Mail": "",
-	"Date":  "02.09.2015", "Comment": "Created mview" },
-	{"Name": "Ludee", "Mail": "",
-	"Date":  "17.12.2016", "Comment": "Added metadata" } ],
-    "Notes": [""],
-    "Licence": [{
-	"Name": "", 
-	"URL": "" }],
-    "Instructions for proper use": [" "]
-    }' ;
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
 
 -- select description
 SELECT obj_description('model_draft.ego_demand_la_zensus' ::regclass) ::json;
@@ -113,8 +91,9 @@ CREATE TABLE         	model_draft.ego_demand_la_zensus_cluster (
 
 -- insert cluster
 INSERT INTO	model_draft.ego_demand_la_zensus_cluster(geom)
-	SELECT	(ST_DUMP(ST_MULTI(ST_UNION(grid.geom)))).geom ::geometry(Polygon,3035) AS geom
-	FROM    model_draft.ego_demand_la_zensus AS grid;
+	SELECT	(ST_DUMP(ST_MULTI(ST_UNION(geom)))).geom ::geometry(Polygon,3035)
+	FROM    model_draft.ego_demand_la_zensus
+    ORDER BY gid;
 
 -- index gist (geom)
 CREATE INDEX	ego_demand_la_zensus_cluster_geom_idx
@@ -156,32 +135,9 @@ ALTER TABLE	model_draft.ego_demand_la_zensus_cluster OWNER TO oeuser;
 
 -- metadata
 COMMENT ON TABLE model_draft.ego_demand_la_zensus_cluster IS '{
-    "Name": "ego zensus loads cluster",
-    "Source":   [{
-	"Name": "open_eGo",
-	"URL": "https://github.com/openego/data_processing"}],
-    "Reference date": "2016",
-    "Date of collection": "02.09.2016",
-    "Original file": ["ego_grid_hvmv_substation"],
-    "Spatial": [{
-	"Resolution": "",
-	"Extend": "Germany" }],
-    "Description": ["osm laods"],
-    "Column":[
-        {"Name": "id", "Description": "Unique identifier", "Unit": " " },
-        {"Name": "area_ha", "Description": "Area", "Unit": "ha" },
-	{"Name": "geom", "Description": "Geometry", "Unit": " " } ],
-    "Changes":	[
-        {"Name": "Ludee", "Mail": "",
-	"Date":  "02.09.2015", "Comment": "Created mview" },
-	{"Name": "Ludee", "Mail": "",
-	"Date":  "17.12.2016", "Comment": "Added metadata" } ],
-    "Notes": [""],
-    "Licence": [{
-	"Name": "", 
-	"URL": "" }],
-    "Instructions for proper use": [" "]
-    }' ;
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
 
 -- select description
 SELECT obj_description('model_draft.ego_demand_la_zensus_cluster' ::regclass) ::json;
@@ -198,6 +154,11 @@ CREATE MATERIALIZED VIEW         	model_draft.ego_society_zensus_per_la_mview AS
 		count(geom) AS census_count
 	FROM	society.destatis_zensus_population_per_ha_mview
 	UNION ALL 
+	SELECT 	'destatis_zensus_population_per_ha_invg_mview' AS name,
+		sum(population), 
+		count(geom) AS census_count
+	FROM	model_draft.destatis_zensus_population_per_ha_invg_mview
+	UNION ALL 
 	SELECT 	'ego_demand_la_zensus' AS name,
 		sum(population), 
 		count(geom) AS census_count
@@ -211,5 +172,11 @@ CREATE MATERIALIZED VIEW         	model_draft.ego_society_zensus_per_la_mview AS
 -- grant (oeuser)
 ALTER TABLE	model_draft.ego_society_zensus_per_la_mview OWNER TO oeuser;
 
+-- metadata
+COMMENT ON TABLE model_draft.ego_society_zensus_per_la_mview IS '{
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
+
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_society_zensus_per_la_mview','ego_dp_loadarea_census.sql',' ');

dataprocessing/sql_snippets/ego_dp_loadarea_loadcut_griddistrict.sql

@@ -100,6 +100,15 @@ UPDATE 	model_draft.ego_demand_loadarea AS t1
 		) AS t2
 	WHERE  	t1.id = t2.id;
 
+-- metadata
+COMMENT ON TABLE model_draft.ego_demand_loadarea IS '{
+    "comment": "eGoDP - Versioning table", 
+    "version": "v0.3.0",
+    "published": "demand.ego_dp_loadarea" }';
+
+-- ego scenario log (version,io,schema_name,table_name,script_name,comment)
+SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_demand_loadarea','ego_dp_loadarea_loadcut_griddistrict.sql',' ');
+
 
 -- validate area (area_ha) -> exclude smaller 100m²
 DROP MATERIALIZED VIEW IF EXISTS	model_draft.ego_demand_loadarea_smaller100m2_mview CASCADE;
@@ -121,6 +130,12 @@ CREATE INDEX	ego_demand_loadarea_smaller100m2_mview_geom_idx
 -- grant (oeuser)
 ALTER TABLE	model_draft.ego_demand_loadarea_smaller100m2_mview OWNER TO oeuser;
 
+-- metadata
+COMMENT ON MATERIALIZED VIEW model_draft.ego_demand_loadarea_smaller100m2_mview IS '{
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
+
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','temp','model_draft','ego_demand_loadarea_smaller100m2_mview','ego_dp_loadarea_loadcut_griddistrict.sql',' ');
 
@@ -185,22 +200,22 @@ UPDATE 	model_draft.ego_demand_loadarea AS t1
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','input','society','destatis_zensus_population_per_ha_mview','ego_dp_loadarea_loadcut_griddistrict.sql',' ');
 
--- zensus 2011 population
-UPDATE 	model_draft.ego_demand_loadarea AS t1
-	SET  	zensus_sum = t2.zensus_sum,
-		zensus_count = t2.zensus_count,
-		zensus_density = t2.zensus_density
-	FROM    (SELECT	a.id AS id,
-			SUM(b.population)::integer AS zensus_sum,
-			COUNT(b.geom_point)::integer AS zensus_count,
-			(SUM(b.population)/COUNT(b.geom_point))::numeric AS zensus_density
-		FROM	model_draft.ego_demand_loadarea AS a,
-			society.destatis_zensus_population_per_ha_mview AS b
-		WHERE  	a.geom && b.geom_point AND
-			ST_CONTAINS(a.geom,b.geom_point)
-		GROUP BY a.id
-		)AS t2
-	WHERE  	t1.id = t2.id;
+-- zensus 2011 population in LA
+UPDATE model_draft.ego_demand_loadarea AS t1
+    SET zensus_sum = t2.zensus_sum,
+        zensus_count = t2.zensus_count,
+        zensus_density = t2.zensus_density
+    FROM (  SELECT  a.id AS id,
+                    SUM(b.population) ::integer AS zensus_sum,
+                    COUNT(b.geom_point) ::integer AS zensus_count,
+                    (SUM(b.population)/COUNT(b.geom_point)) ::numeric AS zensus_density
+            FROM    model_draft.ego_demand_loadarea AS a,
+                    society.destatis_zensus_population_per_ha_mview AS b
+            WHERE   a.geom && b.geom_point AND
+                    ST_CONTAINS(a.geom,b.geom_point)
+            GROUP BY a.id
+        )AS t2
+    WHERE  	t1.id = t2.id;
 
 /*
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
@@ -269,6 +284,12 @@ UPDATE 	model_draft.ego_demand_loadarea AS t1
 		) AS t2
 	WHERE  	t1.id = t2.id;
 
+-- metadata
+COMMENT ON TABLE model_draft.ego_osm_sector_per_griddistrict_1_residential IS '{
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
+
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_osm_sector_per_griddistrict_1_residential','ego_dp_loadarea_loadcut_griddistrict.sql',' ');
 
@@ -318,6 +339,12 @@ UPDATE 	model_draft.ego_demand_loadarea AS t1
 		) AS t2
 	WHERE  	t1.id = t2.id;
 
+-- metadata
+COMMENT ON TABLE model_draft.ego_osm_sector_per_griddistrict_2_retail IS '{
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
+
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_osm_sector_per_griddistrict_2_retail','ego_dp_loadarea_loadcut_griddistrict.sql',' ');
 
@@ -345,6 +372,12 @@ CREATE INDEX  	osm_deu_polygon_urban_sector_3_industrial_nolargescale_mview_geom
 -- grant (oeuser)
 ALTER TABLE	openstreetmap.osm_deu_polygon_urban_sector_3_industrial_nolargescale_mview OWNER TO oeuser;
 
+-- metadata
+COMMENT ON MATERIALIZED VIEW model_draft.osm_deu_polygon_urban_sector_3_industrial_nolargescale_mview IS '{
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
+
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','output','openstreetmap','osm_deu_polygon_urban_sector_3_industrial_nolargescale_mview','setup_osm_landuse.sql',' ');
 
@@ -404,6 +437,12 @@ UPDATE 	model_draft.ego_demand_loadarea AS t1
 		) AS t2
 	WHERE  	t1.id = t2.id;
 
+-- metadata
+COMMENT ON TABLE model_draft.ego_osm_sector_per_griddistrict_3_industrial IS '{
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
+
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_osm_sector_per_griddistrict_3_industrial','ego_dp_loadarea_loadcut_griddistrict.sql',' ');
 
@@ -453,6 +492,12 @@ UPDATE 	model_draft.ego_demand_loadarea AS t1
 		) AS t2
 	WHERE  	t1.id = t2.id;
 
+-- metadata
+COMMENT ON TABLE model_draft.ego_osm_sector_per_griddistrict_4_agricultural IS '{
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
+
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','output','model_draft','ego_osm_sector_per_griddistrict_4_agricultural','ego_dp_loadarea_loadcut_griddistrict.sql',' ');
 
@@ -560,6 +605,12 @@ CREATE INDEX	ego_demand_loadarea_error_noags_mview_geom_idx
 -- grant (oeuser)
 ALTER TABLE	model_draft.ego_demand_loadarea_error_noags_mview OWNER TO oeuser;
 
+-- metadata
+COMMENT ON MATERIALIZED VIEW model_draft.ego_demand_loadarea_error_noags_mview IS '{
+    "comment": "eGoDP - Temporary table", 
+    "version": "v0.3.0",
+    "published": "none" }';
+
 -- ego scenario log (version,io,schema_name,table_name,script_name,comment)
 SELECT ego_scenario_log('v0.3.0','temp','model_draft','ego_demand_loadarea_error_noags_mview','ego_dp_loadarea_loadcut_griddistrict.sql',' ');