From 7fac9fd001e257ed097809029fae132da0a2669a Mon Sep 17 00:00:00 2001 From: vortex Date: Tue, 20 Jan 2015 14:34:47 +0100 Subject: [PATCH 01/39] added processing and publishing earthstat data --- .../demo/ghg/data/update/update_metadata.py | 16 +- geobricks_data_scripts/dev/config/config.py | 18 +- geobricks_data_scripts/dev/data/__init__.py | 1 + .../dev/data/earthstat/__init__.py | 1 + .../dev/data/earthstat/delete/__init__.py | 1 + .../earthstat/delete/delete_all_earthstat.py | 12 ++ .../dev/data/earthstat/processing/__init__.py | 1 + .../processing/processing_earthstat_data.py | 200 ++++++++++++++++++ .../earthstat/processing/publish_harvest.py | 12 ++ .../dev/data/earthstat/publish/__init__.py | 1 + .../dev/data/ghg/__init__.py | 1 + .../dev/data/ghg/delete/__init__.py | 1 + .../delete/data_manager_delete_all_example.py | 12 ++ .../dev/data/ghg/delete/delete_a_layer.py | 7 + .../dev/data/ghg/process_folders/__init__.py | 1 + .../rename_burned_areas_folders_and_files.py | 97 +++++++++ .../dev/data/ghg/publish/__init__.py | 1 + .../dev/data/ghg/publish/publish_harvest.py | 87 ++++++++ .../dev/data/ghg/update/__init__.py | 1 + .../dev/data/ghg/update/style_update.py | 1 + .../data/delete/delete_storage_metadata.py | 17 +- .../data/publish/publish_storage_metadata.py | 2 +- .../utils/harvest/harvest_rasters.py | 73 ++++--- .../utils/harvest/publish_harvest.py | 133 ++++++++++++ 24 files changed, 660 insertions(+), 37 deletions(-) create mode 100644 geobricks_data_scripts/dev/data/__init__.py create mode 100644 geobricks_data_scripts/dev/data/earthstat/__init__.py create mode 100644 geobricks_data_scripts/dev/data/earthstat/delete/__init__.py create mode 100644 geobricks_data_scripts/dev/data/earthstat/delete/delete_all_earthstat.py create mode 100644 geobricks_data_scripts/dev/data/earthstat/processing/__init__.py create mode 100644 geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py create mode 100644 geobricks_data_scripts/dev/data/earthstat/processing/publish_harvest.py create mode 100644 geobricks_data_scripts/dev/data/earthstat/publish/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/delete/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py create mode 100644 geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py create mode 100644 geobricks_data_scripts/dev/data/ghg/process_folders/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/process_folders/rename_burned_areas_folders_and_files.py create mode 100644 geobricks_data_scripts/dev/data/ghg/publish/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py create mode 100644 geobricks_data_scripts/dev/data/ghg/update/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/update/style_update.py create mode 100644 geobricks_data_scripts/utils/harvest/publish_harvest.py diff --git a/geobricks_data_scripts/demo/ghg/data/update/update_metadata.py b/geobricks_data_scripts/demo/ghg/data/update/update_metadata.py index 800bce1..2cf1770 100644 --- a/geobricks_data_scripts/demo/ghg/data/update/update_metadata.py +++ b/geobricks_data_scripts/demo/ghg/data/update/update_metadata.py @@ -12,6 +12,7 @@ def update_dsd_with_wms_and_datasource(wms_url, datasource): dsd = l["dsd"] dsd["datasource"] = datasource print l["uid"], dsd + update_dsd_with_epsg_in_layername(l["uid"]) #update_dsd_with_wms_and_datasource_by_uid(l["uid"], wms_url, datasource) @@ -24,6 +25,19 @@ def update_dsd_with_wms_and_datasource_by_uid(uid, wms_url, datasource): print result +def update_dsd_with_epsg_in_layername(uid, epsg="3857"): + layer = data_manager.metadata_manager.get_by_uid(uid, False, True) + dsd = layer["dsd"] + print dsd + #dsd["layerName"] = dsd["layerName"] + "_" + epsg + # print dsd["layerName"] + # dsd["layerName"] = dsd["layerName"].replace("_3857", "") + # print dsd["layerName"] + #print dsd + #result = data_manager.metadata_manager.overwrite_dsd_rid(dsd) + #print result + + -update_dsd_with_wms_and_datasource("http://fenix.fao.org/geoserver", "geoserver") +update_dsd_with_wms_and_datasource("//fenix.fao.org/geoserver", "geoserver") # update_dsd_with_wms_and_datasource_by_uid("mod13a2", "http://fenix.fao.org/geoserver", "geoserver") \ No newline at end of file diff --git a/geobricks_data_scripts/dev/config/config.py b/geobricks_data_scripts/dev/config/config.py index 791ef79..aa56fde 100644 --- a/geobricks_data_scripts/dev/config/config.py +++ b/geobricks_data_scripts/dev/config/config.py @@ -41,7 +41,7 @@ # Geoserver "geoserver": { - "geoserver_master": "http://lprapp16.fao.org:20100/geoserver/rest", + "geoserver_master": "http://localhost:9090/geoserver/rest", "geoserver_slaves": [], "username": "fenix", "password": "Fenix2014", @@ -49,22 +49,22 @@ # Metadata "metadata": { - "url_create_metadata": "http://fenix.fao.org/d3s_dev/msd/resources/metadata", - "url_get_metadata_uid": "http://fenix.fao.org/d3s_dev/msd/resources/metadata/uid/", + "url_create_metadata": "http://localhost:7788/v2/msd/resources/metadata", + "url_get_metadata_uid": "http://localhost:7788/v2/msd/resources/metadata/uid/", # delete metadata - "url_delete_metadata": "http://fenix.fao.org/d3s_dev/msd/resources/metadata/uid/", + "url_delete_metadata": "http://localhost:7788/v2/msd/resources/metadata/uid/", # get metadata - "url_get_metadata": "http://fenix.fao.org/d3s_dev/msd/resources/find", - "url_get_metadata_full": "http://fenix.fao.org/d3s_dev/msd/resources/metadata/uid/?full=true&dsd=true", + "url_get_metadata": "http://localhost:7788/v2/msd/resources/find", + "url_get_metadata_full": "http://localhost:7788/v2/msd/resources/metadata/uid/?full=true&dsd=true", # coding system - "url_create_coding_system": "http://fenix.fao.org/d3s_dev/msd/resources", - "url_data_coding_system": "http://fenix.fao.org/d3s_dev/msd/resources/data/uid/", + "url_create_coding_system": "http://localhost:7788/v2/msd/resources", + "url_data_coding_system": "http://localhost:7788/v2/msd/resources/data/uid/", # DSD - "url_overwrite_dsd_rid": "http://fenix.fao.org/d3s_dev/msd/resources/dsd" + "url_overwrite_dsd_rid": "http://localhost:7788/v2/msd/resources/dsd" } } } diff --git a/geobricks_data_scripts/dev/data/__init__.py b/geobricks_data_scripts/dev/data/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/earthstat/__init__.py b/geobricks_data_scripts/dev/data/earthstat/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/earthstat/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/earthstat/delete/__init__.py b/geobricks_data_scripts/dev/data/earthstat/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/earthstat/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/dev/data/earthstat/delete/delete_all_earthstat.py new file mode 100644 index 0000000..b4e48cb --- /dev/null +++ b/geobricks_data_scripts/dev/data/earthstat/delete/delete_all_earthstat.py @@ -0,0 +1,12 @@ + +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +for layer in layers: + try: + if "earthstat" in layer["dsd"]["workspace"]: + data_manager.delete(layer["uid"]) + except Exception, e: + print e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/__init__.py b/geobricks_data_scripts/dev/data/earthstat/processing/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/earthstat/processing/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py b/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py new file mode 100644 index 0000000..09fb8b0 --- /dev/null +++ b/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py @@ -0,0 +1,200 @@ +import glob +import os +import shutil +from geobricks_common.core.log import logger +from geobricks_common.core.filesystem import get_filename +from geobricks_processing.core.processing_core import process_obj + +log = logger(__file__) + +output_extension = "geotiff" + + +process_obj_3857 = [ + { + # "source_path": [source_path], + # "output_path": output_path, + #"output_file_name": "burundi_maize_area_4326.tif", + "band": 1, + "tmp": True, + "process": [ + { + "gdalwarp": { + "opt": { + "-multi": "", + "-overwrite": "", + "-of": "GTiff", + "-srcnodata": "8.999999828524175e+20", + "-dstnodata": "-3000", + "-s_srs": "EPSG:4326", + "-t_srs": "EPSG:3857" + } + } + } + ] + }, + { + # "source_path": [source_path], + # "output_path": output_path, + # "output_file_name": "burundi_maize_area_4326.tif", + "band": 1, + "process": [ + { + "gdal_translate": { + "opt": { + "-of": "GTiff", + "-a_nodata": "-3000", + "-a_srs": "EPSG:3857", + "-co": "'TILED=YES'", + "-co": "'COMPRESS=DEFLATE'", + } + } + }, + { + "gdaladdo": { + "parameters": { + # "--config": "BIGTIFF_OVERVIEW IF_NEEDED" + }, + "overviews_levels": "2 4 8 16" + } + } + ] + }, + # { + # "band": 1, + # "process": [ + # { + # "gdaladdo": { + # "parameters": { + # # "--config": "BIGTIFF_OVERVIEW IF_NEEDED" + # }, + # "overviews_levels": "2 4 8 16" + # } + # } + # ] + # } +] + + +process_obj_4326 = [ + { + "band": 1, + "process": [ + { + "gdalwarp": { + "opt": { + "-multi": "", + "-overwrite": "", + "-of": "GTiff", + "-srcnodata": "8.999999828524175e+20", + "-dstnodata": "-3000", + "-s_srs": "EPSG:4326", + "-t_srs": "EPSG:4326" + } + } + } + ] + }, + { + "band": 1, + "process": [ + { + "gdal_translate": { + "opt": { + "-of": "GTiff", + "-a_nodata": "-3000", + "-a_srs": "EPSG:4326", + "-co": "'COMPRESS=DEFLATE'", + } + } + } + ] + }, +] + +def process(src_folder, dest_folder="", folder_type="geoserver", prj="3857", overwrite=True, file_type="tif"): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + # dest_folder = create_output_folder(folder, folder_type, overwrite) + files = glob.glob(os.path.join(folder, "*." + file_type)) + for f in files: + if "area" in f: + output_path = os.path.join(dest_folder, "earthstat_crop_area") + elif "yield" in f: + output_path = os.path.join(dest_folder, "earthstat_crop_yield") + + output_path = create_output_folder(output_path, folder_type, overwrite) + + filename = get_filename(f) + if filename[len(filename)-1] == "1": + filename = filename[:-1] + output_file_name = filename + "_" + prj + ".geotiff" + # TODO: switch between processes + print "Processing:", output_file_name + if prj == "3857": + # TODO: worksaround for the various sourcepath e dstpath + process_obj_3857[0]["source_path"] = [f] + process_obj_3857[1]["output_path"] = output_path + process_obj_3857[1]["output_file_name"] = output_file_name + for p in process_obj_3857: + p["source_path"] = p["source_path"] if "source_path" in p else result + result = process_obj(p) + elif prj == "4326": + process_obj_4326[0]["source_path"] = [f] + process_obj_4326[1]["output_path"] = output_path + process_obj_4326[1]["output_file_name"] = output_file_name + for p in process_obj_4326: + p["source_path"] = p["source_path"] if "source_path" in p else result + result = process_obj(p) + + +def create_output_folder(folder, folder_type="geoserver", overwrite=False): + dest_folder = os.path.join(folder, folder_type) + # if folder exists remove it and create if again + # if overwrite: + # if os.path.exists(dest_folder): + # shutil.rmtree(dest_folder) + # if not os.path.exists(dest_folder): + # os.makedirs(dest_folder) + return dest_folder + + +def move_storage_folder(base_folder, file_type="geotiff"): + files = glob.glob(os.path.join(base_folder, "*." + file_type)) + + for f in files: + print f + if os.path.isfile(f): + print f + filename = get_filename(f) + folder = os.path.join(base_folder, filename) + if not os.path.isdir(folder): + os.makedirs(folder) + print folder + # move file to folder + shutil.move(f, folder) + + + + +# load scripts +if __name__ == '__main__': + src_folder = "/home/vortex/Desktop/LAYERS/earthstat/175CropsYieldArea_geotiff/" + dest_folder = "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/" + + + #process(src_folder, dest_folder, "geoserver", "3857") + #process(src_folder, dest_folder, "storage", "4326") + + #move_storage_folder(os.path.join(dest_folder, "earthstat_crop_area/storage/")) + #move_storage_folder(os.path.join(dest_folder, "earthstat_crop_yield/storage/")) + + # N.B. the final move has been made by hand + # earthstat/earthstat_processeddata/ + # earthstat/earthstat_processeddata/geoserver/ + # earthstat/earthstat_processeddata/geoserver/earthstat_crop_area/ + # earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/ + # earthstat/earthstat_processeddata/storage/earthstat_crop_area/ + # earthstat/earthstat_processeddata/storage/earthstat_crop_area/abacaarea_4326/ + # etc... \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/publish_harvest.py b/geobricks_data_scripts/dev/data/earthstat/processing/publish_harvest.py new file mode 100644 index 0000000..9f723d9 --- /dev/null +++ b/geobricks_data_scripts/dev/data/earthstat/processing/publish_harvest.py @@ -0,0 +1,12 @@ +import glob +import os +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style + + +data_manager = get_data_manager() + +lang = "EN" +workspace = "earthstat" + +harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/", workspace) \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/earthstat/publish/__init__.py b/geobricks_data_scripts/dev/data/earthstat/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/earthstat/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/__init__.py b/geobricks_data_scripts/dev/data/ghg/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/delete/__init__.py b/geobricks_data_scripts/dev/data/ghg/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py new file mode 100644 index 0000000..b58c736 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py @@ -0,0 +1,12 @@ + +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +for layer in layers: + try: + print layer["uid"] + #data_manager.delete(layer["uid"]) + except Exception, e: + print e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py b/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py new file mode 100644 index 0000000..f805fee --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py @@ -0,0 +1,7 @@ + +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() + +data_manager.delete("ghg:dm_burning_savanna_2002_3857") +data_manager.delete("ghg:ch4_emissions_burning_otherforests_2004_3857") diff --git a/geobricks_data_scripts/dev/data/ghg/process_folders/__init__.py b/geobricks_data_scripts/dev/data/ghg/process_folders/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/process_folders/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/process_folders/rename_burned_areas_folders_and_files.py b/geobricks_data_scripts/dev/data/ghg/process_folders/rename_burned_areas_folders_and_files.py new file mode 100644 index 0000000..961b6b7 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/process_folders/rename_burned_areas_folders_and_files.py @@ -0,0 +1,97 @@ +import os +import glob + +def rename_burned_areas_folders(folder): + folders = glob.glob(os.path.join(folder, "*")) + print folders + for folder in folders: + print folder + if os.path.isdir(folder): + product_code = os.path.basename(os.path.normpath(folder)) + print product_code + product_name = burned_areas_folder(product_code) + + print + + +def burned_areas_folder(folder): + filename = folder.lower() + if "peatlands" in filename: + return "GFED4 Burned Areas - Organic soils" + if "AllForestsMinus".lower() in filename: + return "GFED4 Burned Areas - Other forest" + if "humidtropicalforests".lower() in filename: + return "GFED4 Burned Areas - Humid Tropical Forest" + + burned_areas_lc = "GFED4 Burned Areas - " + if "1" in filename: + return burned_areas_lc + "Evergreen Needleleaf forest" + if "2" in filename: + return burned_areas_lc + "Evergreen Broadleaf forest" + if "3" in filename: + return burned_areas_lc + "Deciduous Needleleaf forest" + if "4" in filename: + return burned_areas_lc + "Deciduous Broadleaf forest" + if "5" in filename: + return burned_areas_lc + "Mixed forest" + if "6" in filename: + return burned_areas_lc + "Closed shrubland" + if "7" in filename: + return burned_areas_lc + "Open shrubland" + if "8" in filename: + return burned_areas_lc + "Woody savanna" + if "9" in filename: + return burned_areas_lc + "Savanna" + if "10" in filename: + return burned_areas_lc + "Grassland" + if "12" in filename: + return burned_areas_lc + "Croplands" + if "13" in filename: + return burned_areas_lc + "Urban and built-up" + if "16" in filename: + return burned_areas_lc + "Barren or sparsely vegetated" + if "17" in filename: + return burned_areas_lc + "Unclassified" + +def burned_areas_switch(filename): + filename = filename.lower() + if "peatlands" in filename: + return "GFED4 Burned Areas - Organic soils" + if "AllForestsMinus".lower() in filename: + return "GFED4 Burned Areas - Other forest" + if "humidtropicalforests".lower() in filename: + return "GFED4 Burned Areas - Humid Tropical Forest" + + burned_areas_lc = "GFED4 Burned Areas - " + if "lc_1_" in filename: + return burned_areas_lc + "Evergreen Needleleaf forest" + if "lc_2_" in filename: + return burned_areas_lc + "Evergreen Broadleaf forest" + if "lc_3_" in filename: + return burned_areas_lc + "Deciduous Needleleaf forest" + if "lc_4_" in filename: + return burned_areas_lc + "Deciduous Broadleaf forest" + if "lc_5_" in filename: + return burned_areas_lc + "Mixed forest" + if "lc_6_" in filename: + return burned_areas_lc + "Closed shrubland" + if "lc_7_" in filename: + return burned_areas_lc + "Open shrubland" + if "lc_8_" in filename: + return burned_areas_lc + "Woody savanna" + if "lc_9_" in filename: + return burned_areas_lc + "Savanna" + if "lc_10_" in filename: + return burned_areas_lc + "Grassland" + if "lc_12_" in filename: + return burned_areas_lc + "Croplands" + if "lc_13_" in filename: + print filename + return burned_areas_lc + "Urban and built-up" + if "lc_16_" in filename: + return burned_areas_lc + "Barren or sparsely vegetated" + if "lc_17_" in filename: + return burned_areas_lc + "Unclassified" + + +rename_burned_areas_folders("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GFED4_BURNEDAREAS_BY_LANDCOVER/") \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/publish/__init__.py b/geobricks_data_scripts/dev/data/ghg/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py b/geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py new file mode 100644 index 0000000..93440fd --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py @@ -0,0 +1,87 @@ +import glob +import os +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style + + +data_manager = get_data_manager() + +lang = "EN" +workspace = "ghg" + +def get_maghg_defaultstyle_name(filename): + sldname = "" + if "FUELBIOMASS" in filename.upper(): + sldname = "fuelbiomass" + elif "CH4_EF" in filename.upper(): + sldname = "ch4_ef" + elif "CO2_EF" in filename.upper(): + sldname = "co2_ef" + elif "N2O_EF" in filename.upper(): + sldname = "n2o_ef" + elif "CH4" in filename.upper(): + sldname = "ch4" + elif "CO2" in filename.upper(): + sldname = "co2" + elif "DM" in filename.upper(): + sldname = "dm" + elif "N2O" in filename.upper(): + sldname = "n2o" + return sldname + "_emissions_burning_" + lang + +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/Cultivation_organic_soils_croplands/Cultivation_organic_soils_-_croplands/", workspace) +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GriddedLivestock/gridded_livestock_of_the_world_v_201/", workspace) +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/JRC_CLIMATE_ZONE/JRC_climate_zone/", workspace) +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/glc2000/glc2000_v1/", workspace) +# harvest_folder(data_manager, "/home/vortex/Deskt/op/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GEZ/global_ecological_zones_GEZ/", workspace) + + +# GHG-MAG +# 4326 +def harvest_maghg_4326(): + maghg_4326_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/storage" + folders = glob.glob(os.path.join(maghg_4326_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + harvest_folder(data_manager, folder, workspace) + +# 3857 +def harvest_maghg_3857(): + maghg_3857_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/geoserver" + folders = glob.glob(os.path.join(maghg_3857_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + harvest_folder(data_manager, folder, workspace) + + +def harvest_maghg_3857_updated_defaultstyle(): + maghg_3857_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/geoserver" + folders = glob.glob(os.path.join(maghg_3857_folder, "*")) + all_metadatas = [] + for folder in folders: + if os.path.isdir(folder): + all_metadatas.append(harvest_folder(data_manager, folder, workspace, False, False, False)) + + # update the metadata retrieved + for metadatas in all_metadatas: + for metadata in metadatas: + try: + print metadata["dsd"]["layerName"] + default_style = get_maghg_defaultstyle_name(metadata["dsd"]["layerName"]) + data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], default_style) + except Exception, e: + print "ERRRROOOOO!!!!!" + print e + + + + + + + + +#harvest_maghg_4326() +harvest_maghg_3857() +harvest_maghg_3857_updated_defaultstyle() + + diff --git a/geobricks_data_scripts/dev/data/ghg/update/__init__.py b/geobricks_data_scripts/dev/data/ghg/update/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/update/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/update/style_update.py b/geobricks_data_scripts/dev/data/ghg/update/style_update.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/update/style_update.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/storage/data/delete/delete_storage_metadata.py b/geobricks_data_scripts/dev/storage/data/delete/delete_storage_metadata.py index 5a3d2c9..d56ccb6 100644 --- a/geobricks_data_scripts/dev/storage/data/delete/delete_storage_metadata.py +++ b/geobricks_data_scripts/dev/storage/data/delete/delete_storage_metadata.py @@ -1,6 +1,19 @@ +from geobricks_common.core.log import logger from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +log = logger(__file__) + data_manager = get_data_manager() +try: + # TODO How to handle the fact that is in storage? + data_manager.delete("ghg:cultivation_organic_soils_-_croplands_3857", True, False, False) + data_manager.delete("cultivation_organic_soils_-_croplands_4326", True, False, False) -# TODO How to handle the fact that is in storage? -data_manager.delete("mod13a2", True, False, False) + data_manager.delete("ghg:sheep10km_ad_2010_v2_2010_3857", True, False, False) + data_manager.delete("sheep10km_ad_2010_v2_2010_4326", True, False, False) + data_manager.delete("ghg:cattle10km_ad_2010_v2_2010_3857", True, False, False) + data_manager.delete("cattle10km_ad_2010_v2_2010_4326", True, False, False) +except Exception, e: + log.error(e) + pass diff --git a/geobricks_data_scripts/dev/storage/data/publish/publish_storage_metadata.py b/geobricks_data_scripts/dev/storage/data/publish/publish_storage_metadata.py index 7fad708..13d7c96 100644 --- a/geobricks_data_scripts/dev/storage/data/publish/publish_storage_metadata.py +++ b/geobricks_data_scripts/dev/storage/data/publish/publish_storage_metadata.py @@ -89,7 +89,7 @@ def publish(input_folder): try: print metadata_def # upload - # data_manager.publish_coveragestore_storage(input_file, metadata_def, False, False, True) + print data_manager.publish_coveragestore_storage(input_file, metadata_def, False, False, True) except Exception, e: print e diff --git a/geobricks_data_scripts/utils/harvest/harvest_rasters.py b/geobricks_data_scripts/utils/harvest/harvest_rasters.py index 18b50c9..6e9044f 100644 --- a/geobricks_data_scripts/utils/harvest/harvest_rasters.py +++ b/geobricks_data_scripts/utils/harvest/harvest_rasters.py @@ -2,12 +2,14 @@ import glob import pyproj import datetime +import simplejson from geobricks_common.core.log import logger from geobricks_common.core.filesystem import get_filename, get_file_extension from geobricks_common.core.date import get_daterange from geobricks_common.core.filesystem import sanitize_name from geobricks_data_scripts.utils.metadata.metadata import create_metadata from geobricks_gis_raster.core.raster import get_authority +from geobricks_common.core.utils import dict_merge log = logger(__file__) @@ -15,30 +17,50 @@ supported_file = ["tif", "tiff", "geotiff", "TIFF", "TIF", "GEOTIFF"] -metadata_file = ["json"] - -def harvest_raster_folder(path, workspace=None): +def harvest_raster_folder(path): # read files + metadatas = [] files = glob.glob(os.path.join(path, "*")) - for f in files: - # sanitize file name? - filename = get_filename(f) - extension = get_file_extension(f) - if extension in supported_file: - log.info(filename) - metadata = parse_filename(filename, get_authority(f).upper()) - metadata = create_metadata(metadata) - - # check if exists a file named like the "file".json ( and in case overwrite the what is used there) - #metadata_file = - print metadata - - if extension in metadata_file: - log.info(filename) + # getting product code from the folder name + product_code = os.path.basename(os.path.normpath(path)) -def parse_filename(filename, map_projection_code): + for f in files: + if os.path.isfile(f): + metadata = None + # sanitize file name? + filename = get_filename(f) + extension = get_file_extension(f) + + # check if is not a json + if extension in supported_file and extension is not "json": + log.info(filename) + metadata = parse_filename(filename, get_authority(f).upper(), product_code) + metadata = create_metadata(metadata) + + # check metadata .json (if exists for the file) with or without projections code at the end + metadata_file_prj = os.path.join(path, filename + ".json") + if os.path.isfile(metadata_file_prj): + with open(metadata_file_prj) as m_file: + d = simplejson.load(m_file) + metadata = dict_merge(metadata, d) + + metadata_file = os.path.join(path, "_".join(filename.split("_")[:(len(filename.split("_"))-1)]) + ".json") + if os.path.isfile(metadata_file): + with open(metadata_file) as m_file: + d = simplejson.load(m_file) + metadata = dict_merge(metadata, d) + + + if metadata: + metadata["path"] = f + metadatas.append(metadata) + return metadatas + + +# the product_code is usually gived by the folder name +def parse_filename(filename, map_projection_code, product_code=None): # get name, date, proj filename = filename s = filename.split("_") @@ -63,9 +85,13 @@ def parse_filename(filename, map_projection_code): s.remove(date) # product (in case it skips the date) - product_code = sanitize_name(' '.join(s)) + if product_code is None: + product_code = sanitize_name(' '.join(s)) + else: + product_code = sanitize_name(product_code) + product_label = sanitize_name(' '.join(s)) - sanitized_title = sanitize_name(title) + sanitized_title = sanitize_name(title) + "_" + prj layerName = sanitized_title @@ -94,9 +120,8 @@ def parse_filename(filename, map_projection_code): return metadata +# TODO: check prj def check_prj(prj): print "Check if prj is valid or is the name" + return True - - -harvest_raster_folder("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/Cultivation_organic_soils_croplands/publish/") diff --git a/geobricks_data_scripts/utils/harvest/publish_harvest.py b/geobricks_data_scripts/utils/harvest/publish_harvest.py new file mode 100644 index 0000000..c9911bd --- /dev/null +++ b/geobricks_data_scripts/utils/harvest/publish_harvest.py @@ -0,0 +1,133 @@ +from geobricks_common.core.log import logger +from geobricks_data_scripts.utils.harvest.harvest_rasters import harvest_raster_folder + +log = logger(__file__) + + +# TODO: apply publish on storage and geoserver +def harvest_folder(data_manager, folder, workspace=None, publish_on_geoserver=True, publish_on_storage=True, update_links=True): + metadatas = harvest_raster_folder(folder) + print metadatas + + # harvest 3857 for publication + for metadata in metadatas: + print metadata + # check epsg + if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": + if "workspace" not in metadata["dsd"]: + metadata["dsd"]["workspace"] = workspace + if publish_on_geoserver: + publish_geoserver(data_manager, metadata["path"], metadata) + + # harvest 4326 (or others) to storage + # it's done after the 3857 it's sure the metadata will be there + for metadata in metadatas: + if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": + if publish_on_storage: + publish_storage(data_manager, metadata["path"], metadata, workspace) + + + if update_links: + # link 3857 layer to 4326 distribution layer + for metadata in metadatas: + print metadata + # check epsg + if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": + update_dsd_layer_to_distribution_layer(data_manager, metadata) + + + # link 4326 layer to 3857 visualization layer + for metadata in metadatas: + print metadata + # check epsg + if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": + update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace) + + return metadatas + +def update_folder_style(data_manager, folder, style=None): + ''' + Updates all layer style in the folder. + the basic defaultStlye is taken by the layer name minus the date (if used) and the prj + :param data_manager: + :param folder: + :param style: + :return: + ''' + metadatas = harvest_raster_folder(folder) + for metadata in metadatas: + if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": + print metadata["dsd"] + try: + if style is not None: + data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], style) + else: + data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], metadata["dsd"]["defaultStyle"]) + except Exception, e: + log.warn(e) + + + +def update_dsd_layer_to_distribution_layer(data_manager, metadata): + update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], "4326", "distribution") + + +def update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace): + update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], "3857", "visualization", workspace) + + +def publish_geoserver(data_manager, path, metadata): + log.info("publish on geoserver") + try: + data_manager.publish_coveragestore(path, metadata) + except Exception, e: + log.error(e) + + +def publish_storage(data_manager, path, metadata, workspace=None, uid_distribution=True): + log.info("publish on storage") + try: + data_manager.publish_coveragestore_storage(path, metadata, False, False, True) + except Exception, e: + log.error(e) + + +def update_dsd_layer_with_layer_dist_or_vis(data_manager, uid, epsg_code="4326", type="distribution", workspace=None): + try: + metadata = data_manager.get_metadata_by_uid(uid) + # TODO this should be done through layername and workspace + layer_uid = "" + if workspace: + layer_uid += workspace + ":" + layer_uid += "_".join(metadata["dsd"]["layerName"].split("_")[:-1]) + "_" + epsg_code + print layer_uid + metadata_layer = data_manager.get_metadata_by_uid(layer_uid) + print metadata_layer + + # add distribution uid + if "contextExtension" not in metadata["dsd"]: + metadata["dsd"]["contextExtension"] = {} + + obj = { + "uid": metadata_layer["uid"] + } + if "version" in metadata_layer: + obj["version"] = metadata_layer["version"] + + # TODO: replace or append? (there should be a policy) + # TODO: For now it's overwrite! + # if type not in metadata["dsd"]["contextExtension"]: + # metadata["dsd"]["contextExtension"][type] = [] + # metadata["dsd"]["contextExtension"][type].append(obj) + metadata["dsd"]["contextExtension"][type] = [obj] + + # update dsd + dsd = metadata["dsd"] + print "UPDATE", dsd + result = data_manager.metadata_manager.overwrite_dsd_rid(dsd) + print result + + except Exception, e: + # log.error("No metadata found for ", str(layer_uid)) + log.error(e) + From 209df951e4d32c0ec962202f6e1ea6688337fa2d Mon Sep 17 00:00:00 2001 From: vortex Date: Tue, 20 Jan 2015 14:44:46 +0100 Subject: [PATCH 02/39] added error --- .../dev/data/earthstat/delete/delete_all_earthstat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geobricks_data_scripts/dev/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/dev/data/earthstat/delete/delete_all_earthstat.py index b4e48cb..4ad344c 100644 --- a/geobricks_data_scripts/dev/data/earthstat/delete/delete_all_earthstat.py +++ b/geobricks_data_scripts/dev/data/earthstat/delete/delete_all_earthstat.py @@ -8,5 +8,5 @@ if "earthstat" in layer["dsd"]["workspace"]: data_manager.delete(layer["uid"]) except Exception, e: - print e + print "Error", e pass \ No newline at end of file From 2ddcf88507643109c1857e1b147a871f01e84d58 Mon Sep 17 00:00:00 2001 From: vortex Date: Tue, 20 Jan 2015 16:39:08 +0100 Subject: [PATCH 03/39] update --- test/test_suite/test_harvest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_suite/test_harvest.py b/test/test_suite/test_harvest.py index 7816d7a..716ab29 100644 --- a/test/test_suite/test_harvest.py +++ b/test/test_suite/test_harvest.py @@ -1,7 +1,7 @@ from geobricks_data_scripts.utils.harvest import harvest_rasters -harvest_rasters("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Climate_Zones_processed/to_publish/") +#harvest_rasters("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Climate_Zones_processed/to_publish/") From 900fe5b92970382ec318e83bcbb510b21a28b402 Mon Sep 17 00:00:00 2001 From: vortex Date: Tue, 20 Jan 2015 16:50:46 +0100 Subject: [PATCH 04/39] k --- test/test_suite/test_harvest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_suite/test_harvest.py b/test/test_suite/test_harvest.py index 716ab29..a6ef091 100644 --- a/test/test_suite/test_harvest.py +++ b/test/test_suite/test_harvest.py @@ -1,6 +1,8 @@ from geobricks_data_scripts.utils.harvest import harvest_rasters + + #harvest_rasters("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Climate_Zones_processed/to_publish/") From 7c0131ab22e097c9beb2e32549b3b1ebbf520452 Mon Sep 17 00:00:00 2001 From: vortex Date: Fri, 23 Jan 2015 16:43:55 +0100 Subject: [PATCH 05/39] geoserver slds --- .../data/earthstat/processing/move_files.py | 1 + .../publish_harvest.py | 0 .../dev/data/ghg/process/__init__.py | 1 + .../data/ghg/process/process_burned_areas.py | 1 + .../dev/data/ghg/publish/publish_GHG_data.py | 61 +++++++++ .../geoserver_sld/burned_areas_EN.sld | 31 +++++ .../geoserver_sld/burned_areas_EN.xml | 8 ++ .../geoserver_sld/cattle10km_ad_v2_EN.sld | 25 ++++ .../geoserver_sld/cattle10km_ad_v2_EN.xml | 8 ++ .../geoserver_sld/ch4_EN.sld | 32 +++++ .../geoserver_sld/ch4_ef_EN.sld | 31 +++++ .../geoserver_sld/co2_EN.sld | 32 +++++ .../geoserver_sld/co2_ef_EN.sld | 31 +++++ ...ltivation_organic_soils_-_croplands_EN.sld | 27 ++++ ...ltivation_organic_soils_-_croplands_EN.xml | 8 ++ .../geoserver_sld/dm_EN.sld | 33 +++++ .../geoserver_sld/earthstat_crop_area_EN.sld | 34 +++++ .../geoserver_sld/earthstat_crop_area_EN.xml | 8 ++ .../geoserver_sld/fuelbiomass_EN.sld | 31 +++++ .../global_ecological_zones_gez_EN.sld | 42 ++++++ .../global_land_cover_glc2000_EN.sld | 39 ++++++ .../geoserver_sld/jrc_climate_zone_EN.sld | 32 +++++ .../geoserver_sld/jrc_climate_zone_EN.xml | 8 ++ geobricks_data_scripts/geoserver_sld/modis | 39 ++++++ .../modis_-_land_cover_type_umd_EN.sld | 39 ++++++ .../modis_-_land_cover_type_umd_EN.xml | 8 ++ .../geoserver_sld/n2o_EN.sld | 34 +++++ .../geoserver_sld/n2o_ef_EN.sld | 31 +++++ .../geoserver_sld/sheep10km_ad_v2_EN.sld | 25 ++++ .../geoserver_sld/sheep10km_ad_v2_EN.xml | 8 ++ .../woodysavanna_fuelbiomass_EN.sld | 31 +++++ .../test/coding_system/__init__.py | 1 + .../test/coding_system/create_cs_csv.py | 48 +++++++ .../test/coding_system/csv/cs_ghg.csv | 25 ++++ .../test/coding_system/csv/cs_maghg.csv | 45 +++++++ .../test/coding_system/csv/final.csv | 68 ++++++++++ .../csv/final_without_maghg_and_soil.csv | 68 ++++++++++ .../test/coding_system/publish_final_cs.py | 43 ++++++ .../test/coding_system/publish_maghg_cs.py | 26 ++++ .../test/common_data/__init__.py | 1 + .../common_data/coding_system/__init__.py | 1 + .../coding_system/update_mapProjections.py | 33 +++++ geobricks_data_scripts/test/config/.gitignore | 1 + .../test/config/__init__.py | 1 + geobricks_data_scripts/test/config/config.py | 77 +++++++++++ geobricks_data_scripts/test/data/__init__.py | 1 + .../test/data/ghg/__init__.py | 1 + .../test/data/ghg/delete/__init__.py | 1 + .../delete/data_manager_delete_all_example.py | 12 ++ .../test/data/ghg/publish/__init__.py | 1 + .../test/data/ghg/publish/publish_GHG_data.py | 61 +++++++++ geobricks_data_scripts/test/utils/.gitignore | 1 + geobricks_data_scripts/test/utils/__init__.py | 1 + .../test/utils/data_manager_util.py | 6 + .../utils/harvest/harvest_rasters._oldpy | 127 ++++++++++++++++++ geobricks_data_scripts/utils/rename.py | 1 + 56 files changed, 1390 insertions(+) create mode 100644 geobricks_data_scripts/dev/data/earthstat/processing/move_files.py rename geobricks_data_scripts/dev/data/earthstat/{processing => publish}/publish_harvest.py (100%) create mode 100644 geobricks_data_scripts/dev/data/ghg/process/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py create mode 100644 geobricks_data_scripts/dev/data/ghg/publish/publish_GHG_data.py create mode 100644 geobricks_data_scripts/geoserver_sld/burned_areas_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burned_areas_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/ch4_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/ch4_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/co2_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/co2_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/dm_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/global_ecological_zones_gez_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/global_land_cover_glc2000_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/modis create mode 100644 geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/n2o_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/n2o_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/woodysavanna_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/test/coding_system/__init__.py create mode 100644 geobricks_data_scripts/test/coding_system/create_cs_csv.py create mode 100644 geobricks_data_scripts/test/coding_system/csv/cs_ghg.csv create mode 100644 geobricks_data_scripts/test/coding_system/csv/cs_maghg.csv create mode 100644 geobricks_data_scripts/test/coding_system/csv/final.csv create mode 100644 geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv create mode 100644 geobricks_data_scripts/test/coding_system/publish_final_cs.py create mode 100644 geobricks_data_scripts/test/coding_system/publish_maghg_cs.py create mode 100644 geobricks_data_scripts/test/common_data/__init__.py create mode 100644 geobricks_data_scripts/test/common_data/coding_system/__init__.py create mode 100644 geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py create mode 100644 geobricks_data_scripts/test/config/.gitignore create mode 100644 geobricks_data_scripts/test/config/__init__.py create mode 100644 geobricks_data_scripts/test/config/config.py create mode 100644 geobricks_data_scripts/test/data/__init__.py create mode 100644 geobricks_data_scripts/test/data/ghg/__init__.py create mode 100644 geobricks_data_scripts/test/data/ghg/delete/__init__.py create mode 100644 geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py create mode 100644 geobricks_data_scripts/test/data/ghg/publish/__init__.py create mode 100644 geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py create mode 100644 geobricks_data_scripts/test/utils/.gitignore create mode 100644 geobricks_data_scripts/test/utils/__init__.py create mode 100644 geobricks_data_scripts/test/utils/data_manager_util.py create mode 100644 geobricks_data_scripts/utils/harvest/harvest_rasters._oldpy create mode 100644 geobricks_data_scripts/utils/rename.py diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py b/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/publish_harvest.py b/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py similarity index 100% rename from geobricks_data_scripts/dev/data/earthstat/processing/publish_harvest.py rename to geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py diff --git a/geobricks_data_scripts/dev/data/ghg/process/__init__.py b/geobricks_data_scripts/dev/data/ghg/process/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/process/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py b/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/dev/data/ghg/publish/publish_GHG_data.py new file mode 100644 index 0000000..a6d7ff2 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/publish/publish_GHG_data.py @@ -0,0 +1,61 @@ +import os +import glob +from geobricks_common.core.log import logger +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style + +log = logger(__file__) + +data_manager = get_data_manager() + +workspace = "test" + + +def default_import(src_folder): + log.info("GLC2000: " + src_folder) + metadatas = harvest_folder(data_manager, src_folder, workspace, True, True, True) + + +def burned_areas(src_folder): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + folder = os.path.join(folder, "output") + log.info(folder) + metadatas = harvest_folder(data_manager, folder, workspace, True, True, True) + log.info(metadatas) + for metadata in metadatas: + log.info(metadata) + if "3857" in metadata["dsd"]["layerName"]: + log.info("here") + # log.info"metadata["dsd"]["datasource"]) + data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], workspace, "burned_areas_EN") + + +# used just to set the style to modis that was wrong +def set_style_modis(): + data_manager.geoserver_manager.set_style("modis_land_cover_mod12q1_umd_500m_2009_3857", workspace, "modis_-_land_cover_type_umd_EN") + + +if __name__ == '__main__': + base_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/TO_PUBLISH" + + #Cultivation organic soils + #default_import(os.path.join(base_folder, "Cultivation_organic_soils_croplands", "Cultivation_organic_soils_-_croplands")) + + # Burned areas + #burned_areas(os.path.join(base_folder, "GFED4_BURNEDAREAS_BY_LANDCOVER")) + + # GLC 2000 + #default_import(os.path.join(base_folder, "Global_Land_Cover_2000_(GLC_2000)", "Global_Land_Cover_2000_(GLC_2000)")) + + # griddedlivestock + #default_import(os.path.join(base_folder, "GriddedLivestock", "gridded_livestock_of_the_world_v_201")) + + # JRC Climate zone + #default_import(os.path.join(base_folder, "JRC_CLIMATE_ZONE", "JRC_climate_zone")) + + # MODIS_-_Land_Cover_type_UMD + #default_import(os.path.join(base_folder, "MODIS_land_cover", "MODIS_-_Land_Cover_type_UMD")) + + + diff --git a/geobricks_data_scripts/geoserver_sld/burned_areas_EN.sld b/geobricks_data_scripts/geoserver_sld/burned_areas_EN.sld new file mode 100644 index 0000000..85e2b1c --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burned_areas_EN.sld @@ -0,0 +1,31 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burned_areas_EN.xml b/geobricks_data_scripts/geoserver_sld/burned_areas_EN.xml new file mode 100644 index 0000000..fb8809b --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burned_areas_EN.xml @@ -0,0 +1,8 @@ + diff --git a/geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.sld b/geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.sld new file mode 100644 index 0000000..84a6e43 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.sld @@ -0,0 +1,25 @@ + + + + + + + + 20120812 + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.xml b/geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.xml new file mode 100644 index 0000000..d508b0c --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.xml @@ -0,0 +1,8 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/ch4_EN.sld b/geobricks_data_scripts/geoserver_sld/ch4_EN.sld new file mode 100644 index 0000000..6663f5c --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/ch4_EN.sld @@ -0,0 +1,32 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/ch4_ef_EN.sld new file mode 100644 index 0000000..91def84 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/ch4_ef_EN.sld @@ -0,0 +1,31 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/co2_EN.sld b/geobricks_data_scripts/geoserver_sld/co2_EN.sld new file mode 100644 index 0000000..e2c4e3a --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/co2_EN.sld @@ -0,0 +1,32 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/co2_ef_EN.sld new file mode 100644 index 0000000..1503fe9 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/co2_ef_EN.sld @@ -0,0 +1,31 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.sld b/geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.sld new file mode 100644 index 0000000..df846f7 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.sld @@ -0,0 +1,27 @@ + + + + + + + + 20120812 + + + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.xml b/geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.xml new file mode 100644 index 0000000..54e76db --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.xml @@ -0,0 +1,8 @@ + diff --git a/geobricks_data_scripts/geoserver_sld/dm_EN.sld b/geobricks_data_scripts/geoserver_sld/dm_EN.sld new file mode 100644 index 0000000..68af6d7 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/dm_EN.sld @@ -0,0 +1,33 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.sld b/geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.sld new file mode 100644 index 0000000..1c79c3e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.sld @@ -0,0 +1,34 @@ + + + + rain + + rain + Rain distribution + + + + 1.0 + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.xml b/geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.xml new file mode 100644 index 0000000..04c4793 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.xml @@ -0,0 +1,8 @@ + diff --git a/geobricks_data_scripts/geoserver_sld/fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/fuelbiomass_EN.sld new file mode 100644 index 0000000..1503fe9 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/fuelbiomass_EN.sld @@ -0,0 +1,31 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/global_ecological_zones_gez_EN.sld b/geobricks_data_scripts/geoserver_sld/global_ecological_zones_gez_EN.sld new file mode 100644 index 0000000..d33fe5e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/global_ecological_zones_gez_EN.sld @@ -0,0 +1,42 @@ + + + + + + + + 20120812 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/global_land_cover_glc2000_EN.sld b/geobricks_data_scripts/geoserver_sld/global_land_cover_glc2000_EN.sld new file mode 100644 index 0000000..fb4ae48 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/global_land_cover_glc2000_EN.sld @@ -0,0 +1,39 @@ + + + + rain + + rain + Rain distribution + + + + 1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.sld b/geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.sld new file mode 100644 index 0000000..9e47944 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.sld @@ -0,0 +1,32 @@ + + + + + + + + 20120812 + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.xml b/geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.xml new file mode 100644 index 0000000..bac24e5 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.xml @@ -0,0 +1,8 @@ + diff --git a/geobricks_data_scripts/geoserver_sld/modis b/geobricks_data_scripts/geoserver_sld/modis new file mode 100644 index 0000000..fb4ae48 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/modis @@ -0,0 +1,39 @@ + + + + rain + + rain + Rain distribution + + + + 1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.sld b/geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.sld new file mode 100644 index 0000000..fb4ae48 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.sld @@ -0,0 +1,39 @@ + + + + rain + + rain + Rain distribution + + + + 1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.xml b/geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.xml new file mode 100644 index 0000000..42234df --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.xml @@ -0,0 +1,8 @@ + diff --git a/geobricks_data_scripts/geoserver_sld/n2o_EN.sld b/geobricks_data_scripts/geoserver_sld/n2o_EN.sld new file mode 100644 index 0000000..fb02cd9 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/n2o_EN.sld @@ -0,0 +1,34 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/n2o_ef_EN.sld new file mode 100644 index 0000000..71f2467 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/n2o_ef_EN.sld @@ -0,0 +1,31 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.sld b/geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.sld new file mode 100644 index 0000000..d113c48 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.sld @@ -0,0 +1,25 @@ + + + + + + + + 20120812 + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.xml b/geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.xml new file mode 100644 index 0000000..48a268f --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.xml @@ -0,0 +1,8 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/woodysavanna_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/woodysavanna_fuelbiomass_EN.sld new file mode 100644 index 0000000..71f2467 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/woodysavanna_fuelbiomass_EN.sld @@ -0,0 +1,31 @@ + + + + + + + + 20120812 + + + name + Feature + + + + geom + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/test/coding_system/__init__.py b/geobricks_data_scripts/test/coding_system/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/coding_system/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/coding_system/create_cs_csv.py b/geobricks_data_scripts/test/coding_system/create_cs_csv.py new file mode 100644 index 0000000..3ff0dde --- /dev/null +++ b/geobricks_data_scripts/test/coding_system/create_cs_csv.py @@ -0,0 +1,48 @@ +import csv + +items = [ + "Cultivation Organic Soils", + "GFED4 Burned Areas", + "GFED4 Burned Areas - Barren or sparsely vegetated", + "GFED4 Burned Areas - Closed shrubland", + "GFED4 Burned Areas - Croplands", + "GFED4 Burned Areas - Deciduous Broadleaf forest", + "GFED4 Burned Areas - Deciduous Needleleaf forest", + "GFED4 Burned Areas - Evergreen Broadleaf forest", + "GFED4 Burned Areas - Evergreen Needleleaf forest", + "GFED4 Burned Areas - Grassland", + "GFED4 Burned Areas - Humid Tropical Forest", + "GFED4 Burned Areas - Mixed forest", + "GFED4 Burned Areas - Open shrubland", + "GFED4 Burned Areas - Organic soils", + "GFED4 Burned Areas - Other forest", + "GFED4 Burned Areas - Savanna", + "GFED4 Burned Areas - Unclassified", + "GFED4 Burned Areas - Urban and built-up", + "GFED4 Burned Areas - Woody savanna", + "Global Ecological Zones (GEZ) 2010", + "Global Land Cover 2000 (GLC2000)", + "Gridded Livestock of the World v. 2.01", + "Harmonized World Soil Database - Organic soils", + "JRC climate zone", + "JRC climate zone (4326)", + "MODIS - Land Cover Type UMD" +] + + +final = [] +with open('csv/cs_ghg.csv', 'wb') as csvfile: + w = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + for i in items: + s = i.lower().replace(" ", "_") + w.writerow([s, i]) + final.append([s, i]) +print final + + + + + + + + diff --git a/geobricks_data_scripts/test/coding_system/csv/cs_ghg.csv b/geobricks_data_scripts/test/coding_system/csv/cs_ghg.csv new file mode 100644 index 0000000..708565a --- /dev/null +++ b/geobricks_data_scripts/test/coding_system/csv/cs_ghg.csv @@ -0,0 +1,25 @@ +cultivation_organic_soils,Cultivation Organic Soils +gfed4_burned_areas,GFED4 Burned Areas +gfed4_burned_areas_-_barren_or_sparsely_vegetated,GFED4 Burned Areas - Barren or sparsely vegetated +gfed4_burned_areas_-_closed_shrubland,GFED4 Burned Areas - Closed shrubland +gfed4_burned_areas_-_croplands,GFED4 Burned Areas - Croplands +gfed4_burned_areas_-_deciduous_broadleaf_forest,GFED4 Burned Areas - Deciduous Broadleaf forest +gfed4_burned_areas_-_deciduous_needleleaf_forest,GFED4 Burned Areas - Deciduous Needleleaf forest +gfed4_burned_areas_-_evergreen_broadleaf_forest,GFED4 Burned Areas - Evergreen Broadleaf forest +gfed4_burned_areas_-_evergreen_needleleaf_forest,GFED4 Burned Areas - Evergreen Needleleaf forest +gfed4_burned_areas_-_grassland,GFED4 Burned Areas - Grassland +gfed4_burned_areas_-_humid_tropical_forest,GFED4 Burned Areas - Humid Tropical Forest +gfed4_burned_areas_-_mixed_forest,GFED4 Burned Areas - Mixed forest +gfed4_burned_areas_-_open_shrubland,GFED4 Burned Areas - Open shrubland +gfed4_burned_areas_-_organic_soils,GFED4 Burned Areas - Organic soils +gfed4_burned_areas_-_other_forest,GFED4 Burned Areas - Other forest +gfed4_burned_areas_-_savanna,GFED4 Burned Areas - Savanna +gfed4_burned_areas_-_unclassified,GFED4 Burned Areas - Unclassified +gfed4_burned_areas_-_urban_and_built-up,GFED4 Burned Areas - Urban and built-up +gfed4_burned_areas_-_woody_savanna,GFED4 Burned Areas - Woody savanna +global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 +global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) +gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 +harmonized_world_soil_database_-_organic_soils,Harmonized World Soil Database - Organic soils +jrc_climate_zone,JRC climate zone +modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD diff --git a/geobricks_data_scripts/test/coding_system/csv/cs_maghg.csv b/geobricks_data_scripts/test/coding_system/csv/cs_maghg.csv new file mode 100644 index 0000000..959a829 --- /dev/null +++ b/geobricks_data_scripts/test/coding_system/csv/cs_maghg.csv @@ -0,0 +1,45 @@ +dm_burning_grasslands,DM Burning Grasslands +ch4_emissions_burning_woodysavanna,CH4 Emissions Burning WoodySavanna +n2o_emissions_burning_grasslands,N2O Emissions Burning Grasslands +n2o_emissions_burning_woodysavanna,N2O Emissions Burning WoodySavanna +co2_emissions_burning_closedshrublands,CO2 Emissions Burning ClosedShrublands +co2_emissions_burning_openshrublands,CO2 Emissions Burning OpenShrublands +co2_emissions_burning_humidtropicalforests,CO2 Emissions Burning HumidTropicalForests +dm_burning_openshrublands,DM Burning OpenShrublands +n2o_emissions_burning_otherforests,N2O Emissions Burning OtherForests +n2o_emissions_burning_humidtropicalforests,N2O Emissions Burning HumidTropicalForests +n2o_emissions_burning_savanna,N2O Emissions Burning Savanna +ch4_emissions_burning_humidtropicalforests,CH4 Emissions Burning HumidTropicalForests +ch4_emissions_burning_peatlands,CH4 Emissions Burning Peatlands +n2o_emissions_burning_peatlands,N2O Emissions Burning Peatlands +ch4_emissions_burning_closedshrublands,CH4 Emissions Burning ClosedShrublands +co2_emissions_burning_grasslands,CO2 Emissions Burning Grasslands +ch4_emissions_burning_grasslands,CH4 Emissions Burning Grasslands +openshrublands_fuelbiomass,OpenShrublands FuelBiomass +co2_emissions_burning_woodysavanna,CO2 Emissions Burning WoodySavanna +ch4_emissions_burning_otherforests,CH4 Emissions Burning OtherForests +dm_burning_closedshrublands,DM Burning ClosedShrublands +dm_burning_woodysavanna,DM Burning WoodySavanna +ch4_emissions_burning_savanna,CH4 Emissions Burning Savanna +co2_emissions_burning_otherforests,CO2 Emissions Burning OtherForests +n2o_emissions_burning_closedshrublands,N2O Emissions Burning ClosedShrublands +co2_emissions_burning_peatlands,CO2 Emissions Burning Peatlands +co2_emissions_burning_savanna,CO2 Emissions Burning Savanna +dm_burning_peatlands,DM Burning Peatlands +dm_burning_humidtropicalforests,DM Burning HumidTropicalForests +savanna_fuelbiomass,Savanna FuelBiomass +n2o_emissions_burning_openshrublands,N2O Emissions Burning OpenShrublands +tropical_forests_ch4_ef,Tropical Forests CH4 EF +dm_burning_savanna,DM Burning Savanna +ch4_emissions_burning_openshrublands,CH4 Emissions Burning OpenShrublands +dm_burning_otherforests,DM Burning OtherForests +tropical_forests_fuelbiomass,Tropical Forests FuelBiomass +woodysavanna_fuelbiomass,WoodySavanna FuelBiomass +grasslands_fuelbiomass,Grasslands FuelBiomass +nontropical_forests_fuelbiomass,NonTropical Forests FuelBiomass +tropical_forests_co2_ef,Tropical Forests CO2 EF +extratropical_forests_n2o_ef,ExtraTropical Forests N2O EF +closedshrublands_fuelbiomass,ClosedShrublands FuelBiomass +tropical_forests_n2o_ef,Tropical Forests N2O EF +extratropical_forests_co2_ef,ExtraTropical Forests CO2 EF +extratropical_forests_ch4_ef,ExtraTropical Forests CH4 EF diff --git a/geobricks_data_scripts/test/coding_system/csv/final.csv b/geobricks_data_scripts/test/coding_system/csv/final.csv new file mode 100644 index 0000000..43b1ff4 --- /dev/null +++ b/geobricks_data_scripts/test/coding_system/csv/final.csv @@ -0,0 +1,68 @@ +cultivation_organic_soils,Cultivation Organic Soils +gfed4_burned_areas_-_barren_or_sparsely_vegetated,GFED4 Burned Areas - Barren or sparsely vegetated +gfed4_burned_areas_-_closed_shrubland,GFED4 Burned Areas - Closed shrubland +gfed4_burned_areas_-_croplands,GFED4 Burned Areas - Croplands +gfed4_burned_areas_-_deciduous_broadleaf_forest,GFED4 Burned Areas - Deciduous Broadleaf forest +gfed4_burned_areas_-_deciduous_needleleaf_forest,GFED4 Burned Areas - Deciduous Needleleaf forest +gfed4_burned_areas_-_evergreen_broadleaf_forest,GFED4 Burned Areas - Evergreen Broadleaf forest +gfed4_burned_areas_-_evergreen_needleleaf_forest,GFED4 Burned Areas - Evergreen Needleleaf forest +gfed4_burned_areas_-_grassland,GFED4 Burned Areas - Grassland +gfed4_burned_areas_-_humid_tropical_forest,GFED4 Burned Areas - Humid Tropical Forest +gfed4_burned_areas_-_mixed_forest,GFED4 Burned Areas - Mixed forest +gfed4_burned_areas_-_open_shrubland,GFED4 Burned Areas - Open shrubland +gfed4_burned_areas_-_organic_soils,GFED4 Burned Areas - Organic soils +gfed4_burned_areas_-_other_forest,GFED4 Burned Areas - Other forest +gfed4_burned_areas_-_savanna,GFED4 Burned Areas - Savanna +gfed4_burned_areas_-_unclassified,GFED4 Burned Areas - Unclassified +gfed4_burned_areas_-_urban_and_built-up,GFED4 Burned Areas - Urban and built-up +gfed4_burned_areas_-_woody_savanna,GFED4 Burned Areas - Woody savanna +global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 +global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) +gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 +jrc_climate_zone,JRC climate zone +modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD +dm_burning_grasslands,DM Burning Grasslands +ch4_emissions_burning_woodysavanna,CH4 Emissions Burning WoodySavanna +n2o_emissions_burning_grasslands,N2O Emissions Burning Grasslands +n2o_emissions_burning_woodysavanna,N2O Emissions Burning WoodySavanna +co2_emissions_burning_closedshrublands,CO2 Emissions Burning ClosedShrublands +co2_emissions_burning_openshrublands,CO2 Emissions Burning OpenShrublands +co2_emissions_burning_humidtropicalforests,CO2 Emissions Burning HumidTropicalForests +dm_burning_openshrublands,DM Burning OpenShrublands +n2o_emissions_burning_otherforests,N2O Emissions Burning OtherForests +n2o_emissions_burning_humidtropicalforests,N2O Emissions Burning HumidTropicalForests +n2o_emissions_burning_savanna,N2O Emissions Burning Savanna +ch4_emissions_burning_humidtropicalforests,CH4 Emissions Burning HumidTropicalForests +ch4_emissions_burning_peatlands,CH4 Emissions Burning Peatlands +n2o_emissions_burning_peatlands,N2O Emissions Burning Peatlands +ch4_emissions_burning_closedshrublands,CH4 Emissions Burning ClosedShrublands +co2_emissions_burning_grasslands,CO2 Emissions Burning Grasslands +ch4_emissions_burning_grasslands,CH4 Emissions Burning Grasslands +openshrublands_fuelbiomass,OpenShrublands FuelBiomass +co2_emissions_burning_woodysavanna,CO2 Emissions Burning WoodySavanna +ch4_emissions_burning_otherforests,CH4 Emissions Burning OtherForests +dm_burning_closedshrublands,DM Burning ClosedShrublands +dm_burning_woodysavanna,DM Burning WoodySavanna +ch4_emissions_burning_savanna,CH4 Emissions Burning Savanna +co2_emissions_burning_otherforests,CO2 Emissions Burning OtherForests +n2o_emissions_burning_closedshrublands,N2O Emissions Burning ClosedShrublands +co2_emissions_burning_peatlands,CO2 Emissions Burning Peatlands +co2_emissions_burning_savanna,CO2 Emissions Burning Savanna +dm_burning_peatlands,DM Burning Peatlands +dm_burning_humidtropicalforests,DM Burning HumidTropicalForests +savanna_fuelbiomass,Savanna FuelBiomass +n2o_emissions_burning_openshrublands,N2O Emissions Burning OpenShrublands +tropical_forests_ch4_ef,Tropical Forests CH4 EF +dm_burning_savanna,DM Burning Savanna +ch4_emissions_burning_openshrublands,CH4 Emissions Burning OpenShrublands +dm_burning_otherforests,DM Burning OtherForests +tropical_forests_fuelbiomass,Tropical Forests FuelBiomass +woodysavanna_fuelbiomass,WoodySavanna FuelBiomass +grasslands_fuelbiomass,Grasslands FuelBiomass +nontropical_forests_fuelbiomass,NonTropical Forests FuelBiomass +tropical_forests_co2_ef,Tropical Forests CO2 EF +extratropical_forests_n2o_ef,ExtraTropical Forests N2O EF +closedshrublands_fuelbiomass,ClosedShrublands FuelBiomass +tropical_forests_n2o_ef,Tropical Forests N2O EF +extratropical_forests_co2_ef,ExtraTropical Forests CO2 EF +extratropical_forests_ch4_ef,ExtraTropical Forests CH4 EF diff --git a/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv b/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv new file mode 100644 index 0000000..43b1ff4 --- /dev/null +++ b/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv @@ -0,0 +1,68 @@ +cultivation_organic_soils,Cultivation Organic Soils +gfed4_burned_areas_-_barren_or_sparsely_vegetated,GFED4 Burned Areas - Barren or sparsely vegetated +gfed4_burned_areas_-_closed_shrubland,GFED4 Burned Areas - Closed shrubland +gfed4_burned_areas_-_croplands,GFED4 Burned Areas - Croplands +gfed4_burned_areas_-_deciduous_broadleaf_forest,GFED4 Burned Areas - Deciduous Broadleaf forest +gfed4_burned_areas_-_deciduous_needleleaf_forest,GFED4 Burned Areas - Deciduous Needleleaf forest +gfed4_burned_areas_-_evergreen_broadleaf_forest,GFED4 Burned Areas - Evergreen Broadleaf forest +gfed4_burned_areas_-_evergreen_needleleaf_forest,GFED4 Burned Areas - Evergreen Needleleaf forest +gfed4_burned_areas_-_grassland,GFED4 Burned Areas - Grassland +gfed4_burned_areas_-_humid_tropical_forest,GFED4 Burned Areas - Humid Tropical Forest +gfed4_burned_areas_-_mixed_forest,GFED4 Burned Areas - Mixed forest +gfed4_burned_areas_-_open_shrubland,GFED4 Burned Areas - Open shrubland +gfed4_burned_areas_-_organic_soils,GFED4 Burned Areas - Organic soils +gfed4_burned_areas_-_other_forest,GFED4 Burned Areas - Other forest +gfed4_burned_areas_-_savanna,GFED4 Burned Areas - Savanna +gfed4_burned_areas_-_unclassified,GFED4 Burned Areas - Unclassified +gfed4_burned_areas_-_urban_and_built-up,GFED4 Burned Areas - Urban and built-up +gfed4_burned_areas_-_woody_savanna,GFED4 Burned Areas - Woody savanna +global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 +global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) +gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 +jrc_climate_zone,JRC climate zone +modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD +dm_burning_grasslands,DM Burning Grasslands +ch4_emissions_burning_woodysavanna,CH4 Emissions Burning WoodySavanna +n2o_emissions_burning_grasslands,N2O Emissions Burning Grasslands +n2o_emissions_burning_woodysavanna,N2O Emissions Burning WoodySavanna +co2_emissions_burning_closedshrublands,CO2 Emissions Burning ClosedShrublands +co2_emissions_burning_openshrublands,CO2 Emissions Burning OpenShrublands +co2_emissions_burning_humidtropicalforests,CO2 Emissions Burning HumidTropicalForests +dm_burning_openshrublands,DM Burning OpenShrublands +n2o_emissions_burning_otherforests,N2O Emissions Burning OtherForests +n2o_emissions_burning_humidtropicalforests,N2O Emissions Burning HumidTropicalForests +n2o_emissions_burning_savanna,N2O Emissions Burning Savanna +ch4_emissions_burning_humidtropicalforests,CH4 Emissions Burning HumidTropicalForests +ch4_emissions_burning_peatlands,CH4 Emissions Burning Peatlands +n2o_emissions_burning_peatlands,N2O Emissions Burning Peatlands +ch4_emissions_burning_closedshrublands,CH4 Emissions Burning ClosedShrublands +co2_emissions_burning_grasslands,CO2 Emissions Burning Grasslands +ch4_emissions_burning_grasslands,CH4 Emissions Burning Grasslands +openshrublands_fuelbiomass,OpenShrublands FuelBiomass +co2_emissions_burning_woodysavanna,CO2 Emissions Burning WoodySavanna +ch4_emissions_burning_otherforests,CH4 Emissions Burning OtherForests +dm_burning_closedshrublands,DM Burning ClosedShrublands +dm_burning_woodysavanna,DM Burning WoodySavanna +ch4_emissions_burning_savanna,CH4 Emissions Burning Savanna +co2_emissions_burning_otherforests,CO2 Emissions Burning OtherForests +n2o_emissions_burning_closedshrublands,N2O Emissions Burning ClosedShrublands +co2_emissions_burning_peatlands,CO2 Emissions Burning Peatlands +co2_emissions_burning_savanna,CO2 Emissions Burning Savanna +dm_burning_peatlands,DM Burning Peatlands +dm_burning_humidtropicalforests,DM Burning HumidTropicalForests +savanna_fuelbiomass,Savanna FuelBiomass +n2o_emissions_burning_openshrublands,N2O Emissions Burning OpenShrublands +tropical_forests_ch4_ef,Tropical Forests CH4 EF +dm_burning_savanna,DM Burning Savanna +ch4_emissions_burning_openshrublands,CH4 Emissions Burning OpenShrublands +dm_burning_otherforests,DM Burning OtherForests +tropical_forests_fuelbiomass,Tropical Forests FuelBiomass +woodysavanna_fuelbiomass,WoodySavanna FuelBiomass +grasslands_fuelbiomass,Grasslands FuelBiomass +nontropical_forests_fuelbiomass,NonTropical Forests FuelBiomass +tropical_forests_co2_ef,Tropical Forests CO2 EF +extratropical_forests_n2o_ef,ExtraTropical Forests N2O EF +closedshrublands_fuelbiomass,ClosedShrublands FuelBiomass +tropical_forests_n2o_ef,Tropical Forests N2O EF +extratropical_forests_co2_ef,ExtraTropical Forests CO2 EF +extratropical_forests_ch4_ef,ExtraTropical Forests CH4 EF diff --git a/geobricks_data_scripts/test/coding_system/publish_final_cs.py b/geobricks_data_scripts/test/coding_system/publish_final_cs.py new file mode 100644 index 0000000..0d975e3 --- /dev/null +++ b/geobricks_data_scripts/test/coding_system/publish_final_cs.py @@ -0,0 +1,43 @@ +import csv +import json + +from geobricks_metadata_manager.core.metadata_manager_d3s_core import MetadataManager +from geobricks_data_scripts.demo.config.config import settings + +metadata_manager = MetadataManager(settings) + +print metadata_manager + +cs = { + "metadata": { + "uid": "layer_products", + "version": "1.0", + "title": { + "EN": "Layers" + }, + "meContent": { + "resourceRepresentationType": "codelist" + } + }, + "data": [ + + ] +} + + +with open('csv/final.csv', 'rb') as csvfile: + rows = csv.reader(csvfile) + for row in rows: + cs["data"].append({ + "code": row[0], + "title": { + "EN": row[1] + } + }) + + +print json.dumps(cs) +print metadata_manager.publish_coding_system(cs, True) + + + diff --git a/geobricks_data_scripts/test/coding_system/publish_maghg_cs.py b/geobricks_data_scripts/test/coding_system/publish_maghg_cs.py new file mode 100644 index 0000000..f4a93fb --- /dev/null +++ b/geobricks_data_scripts/test/coding_system/publish_maghg_cs.py @@ -0,0 +1,26 @@ +import glob +import os +import csv + +files = glob.glob("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/*.tif") + +cached_codes = [] +codes = [] +for f in files: + if "_3857" in f: + if "20" in f or "19" in f: + # it contains a date + folder, filename = os.path.split(f.rsplit("_", 2)[0]) + else: + folder, filename = os.path.split(f.rsplit("_", 1)[0]) + if filename not in cached_codes: + cached_codes.append(filename) + codes.append([filename.lower().replace(" ","_"), filename.replace("_", " ")]) + + +with open('csv/cs_maghg.csv', 'wb') as csvfile: + w = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + for code in codes: + w.writerow(code) + + diff --git a/geobricks_data_scripts/test/common_data/__init__.py b/geobricks_data_scripts/test/common_data/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/common_data/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/common_data/coding_system/__init__.py b/geobricks_data_scripts/test/common_data/coding_system/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/common_data/coding_system/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py b/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py new file mode 100644 index 0000000..c92b6cf --- /dev/null +++ b/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py @@ -0,0 +1,33 @@ +from geobricks_data_scripts.demo.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() + +cs = { + "metadata": { + "uid": "mapProjections", + "version": "1.0" + }, + "data": [ + { + "code": "EPSG:4326", + "title": { + "EN": "Latlong" + } + }, + { + "code": "EPSG:3857", + "title": { + "EN": "Spherical Mercator projection" + } + }, + { + "code": "EPSG:900913", + "title": { + "EN": "Equivalent to Spherical Mercator Projection" + } + } + ] +} + +# Update coding system +data_manager.metadata_manager.update_coding_system(cs, True) \ No newline at end of file diff --git a/geobricks_data_scripts/test/config/.gitignore b/geobricks_data_scripts/test/config/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/geobricks_data_scripts/test/config/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/geobricks_data_scripts/test/config/__init__.py b/geobricks_data_scripts/test/config/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/config/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/config/config.py b/geobricks_data_scripts/test/config/config.py new file mode 100644 index 0000000..109508d --- /dev/null +++ b/geobricks_data_scripts/test/config/config.py @@ -0,0 +1,77 @@ +import logging +import os +import json + +settings = { + + "settings": { + + + # To be used by Flask: DEVELOPMENT ONLY + "debug": True, + + # Flask host: DEVELOPMENT ONLY + "host": "localhost", + + # Flask port: DEVELOPMENT ONLY + "port": 5555, + + # Logging configurations + "logging": { + "level": logging.ERROR, + "format": "%(asctime)s | %(levelname)-8s | %(name)-20s | Line: %(lineno)-5d | %(message)s", + "datefmt": "%d-%m-%Y | %H:%M:%s" + }, + + # Email configurations + "email": { + "settings": "path_to_the_email.json", + "user": None, + "password": None + }, + + # Folders + "folders": { + "config": "config/", + "tmp": "/home/vortex/Desktop/LAYERS/geobricks/tmp/", + "geoserver_datadir": "/home/vortex/programs/SERVERS/tomcat_geoservers/geoserver_data_2_5_3/data/", + "distribution": "/home/vortex/Desktop/LAYERS/geobricks/distribution/", + "storage": "/home/vortex/Desktop/LAYERS/geobricks/storage/" + }, + + # Geoserver + "geoserver": { + "geoserver_master": "http://http://fenix.fao.org:20200/geoserver/rest", + "geoserver_slaves": [], + "username": "admin", + "password": "geoserver", + }, + + # Metadata + "metadata": { + "url_create_metadata": "http://localhost:7788/v2/msd/resources/metadata", + "url_get_metadata_uid": "http://localhost:7788/v2/msd/resources/metadata/uid/", + + # delete metadata + "url_delete_metadata": "http://localhost:7788/v2/msd/resources/metadata/uid/", + + # get metadata + "url_get_metadata": "http://localhost:7788/v2/msd/resources/find", + "url_get_metadata_full": "http://localhost:7788/v2/msd/resources/metadata/uid/?full=true&dsd=true", + + # coding system + "url_create_coding_system": "http://localhost:7788/v2/msd/resources", + "url_data_coding_system": "http://localhost:7788/v2/msd/resources/data/uid/", + + # DSD + "url_overwrite_dsd_rid": "http://localhost:7788/v2/msd/resources/dsd" + } + } +} + + +# Setting email adderess from configuration file +def set_email_settings(): + if os.path.isfile(settings["settings"]["email"]["settings"]): + settings["settings"]["email"] = json.loads(open(settings["settings"]["email"]["settings"]).read()) +set_email_settings() \ No newline at end of file diff --git a/geobricks_data_scripts/test/data/__init__.py b/geobricks_data_scripts/test/data/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/data/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/data/ghg/__init__.py b/geobricks_data_scripts/test/data/ghg/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/data/ghg/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/data/ghg/delete/__init__.py b/geobricks_data_scripts/test/data/ghg/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/data/ghg/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py new file mode 100644 index 0000000..934a485 --- /dev/null +++ b/geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py @@ -0,0 +1,12 @@ + +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +for layer in layers: + try: + print layer["uid"] + data_manager.delete(layer["uid"]) + except Exception, e: + print e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/test/data/ghg/publish/__init__.py b/geobricks_data_scripts/test/data/ghg/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/data/ghg/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py new file mode 100644 index 0000000..7edb3fa --- /dev/null +++ b/geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py @@ -0,0 +1,61 @@ +import os +import glob +from geobricks_common.core.log import logger +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style + +log = logger(__file__) + +data_manager = get_data_manager() + +workspace = "test" + + +def default_import(src_folder): + log.info("GLC2000: " + src_folder) + metadatas = harvest_folder(data_manager, src_folder, workspace, True, True, True) + + +def burned_areas(src_folder): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + folder = os.path.join(folder, "output") + log.info(folder) + metadatas = harvest_folder(data_manager, folder, workspace, True, True, True) + log.info(metadatas) + for metadata in metadatas: + log.info(metadata) + if "3857" in metadata["dsd"]["layerName"]: + log.info("here") + # log.info"metadata["dsd"]["datasource"]) + data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], workspace, "burned_areas_EN") + + +# used just to set the style to modis that was wrong +def set_style_modis(): + data_manager.geoserver_manager.set_style("modis_land_cover_mod12q1_umd_500m_2009_3857", workspace, "modis_-_land_cover_type_umd_EN") + + +if __name__ == '__main__': + base_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/TO_PUBLISH" + + #Cultivation organic soils + default_import(os.path.join(base_folder, "Cultivation_organic_soils_croplands", "Cultivation_organic_soils_-_croplands")) + + # Burned areas + burned_areas(os.path.join(base_folder, "GFED4_BURNEDAREAS_BY_LANDCOVER")) + + # GLC 2000 + default_import(os.path.join(base_folder, "Global_Land_Cover_2000_(GLC_2000)", "Global_Land_Cover_2000_(GLC_2000)")) + + # griddedlivestock + default_import(os.path.join(base_folder, "GriddedLivestock", "gridded_livestock_of_the_world_v_201")) + + # JRC Climate zone + default_import(os.path.join(base_folder, "JRC_CLIMATE_ZONE", "JRC_climate_zone")) + + # MODIS_-_Land_Cover_type_UMD + default_import(os.path.join(base_folder, "MODIS_land_cover", "MODIS_-_Land_Cover_type_UMD")) + + + diff --git a/geobricks_data_scripts/test/utils/.gitignore b/geobricks_data_scripts/test/utils/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/geobricks_data_scripts/test/utils/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/geobricks_data_scripts/test/utils/__init__.py b/geobricks_data_scripts/test/utils/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/utils/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/utils/data_manager_util.py b/geobricks_data_scripts/test/utils/data_manager_util.py new file mode 100644 index 0000000..357578c --- /dev/null +++ b/geobricks_data_scripts/test/utils/data_manager_util.py @@ -0,0 +1,6 @@ +from geobricks_data_manager.core.data_manager_core import DataManager +from geobricks_data_scripts.dev.config.config import settings + + # get the default data manager for demo db +def get_data_manager(): + return DataManager(settings) \ No newline at end of file diff --git a/geobricks_data_scripts/utils/harvest/harvest_rasters._oldpy b/geobricks_data_scripts/utils/harvest/harvest_rasters._oldpy new file mode 100644 index 0000000..6e9044f --- /dev/null +++ b/geobricks_data_scripts/utils/harvest/harvest_rasters._oldpy @@ -0,0 +1,127 @@ +import os +import glob +import pyproj +import datetime +import simplejson +from geobricks_common.core.log import logger +from geobricks_common.core.filesystem import get_filename, get_file_extension +from geobricks_common.core.date import get_daterange +from geobricks_common.core.filesystem import sanitize_name +from geobricks_data_scripts.utils.metadata.metadata import create_metadata +from geobricks_gis_raster.core.raster import get_authority +from geobricks_common.core.utils import dict_merge + +log = logger(__file__) + +language = "EN" + +supported_file = ["tif", "tiff", "geotiff", "TIFF", "TIF", "GEOTIFF"] + + +def harvest_raster_folder(path): + # read files + metadatas = [] + files = glob.glob(os.path.join(path, "*")) + + # getting product code from the folder name + product_code = os.path.basename(os.path.normpath(path)) + + for f in files: + if os.path.isfile(f): + metadata = None + # sanitize file name? + filename = get_filename(f) + extension = get_file_extension(f) + + # check if is not a json + if extension in supported_file and extension is not "json": + log.info(filename) + metadata = parse_filename(filename, get_authority(f).upper(), product_code) + metadata = create_metadata(metadata) + + # check metadata .json (if exists for the file) with or without projections code at the end + metadata_file_prj = os.path.join(path, filename + ".json") + if os.path.isfile(metadata_file_prj): + with open(metadata_file_prj) as m_file: + d = simplejson.load(m_file) + metadata = dict_merge(metadata, d) + + metadata_file = os.path.join(path, "_".join(filename.split("_")[:(len(filename.split("_"))-1)]) + ".json") + if os.path.isfile(metadata_file): + with open(metadata_file) as m_file: + d = simplejson.load(m_file) + metadata = dict_merge(metadata, d) + + + if metadata: + metadata["path"] = f + metadatas.append(metadata) + return metadatas + + +# the product_code is usually gived by the folder name +def parse_filename(filename, map_projection_code, product_code=None): + # get name, date, proj + filename = filename + s = filename.split("_") + prj = s[len(s)-1] + + # check if the last part is the prj + if check_prj(prj): + s.remove(prj) + + # title + title = ' '.join(s) + + # initialize dates + fromdate = None + todate = None + + # check dates + if len(s) > 1: + date = s[len(s)-1] + fromdate, todate = get_daterange(date) + if fromdate is not None and todate is not None: + s.remove(date) + + # product (in case it skips the date) + if product_code is None: + product_code = sanitize_name(' '.join(s)) + else: + product_code = sanitize_name(product_code) + + product_label = sanitize_name(' '.join(s)) + sanitized_title = sanitize_name(title) + "_" + prj + + layerName = sanitized_title + + # it is gived by the product (TOdO: should it be common by all the layers?) + defaultStyle = product_code + "_" + language.upper() + + print "-------------" + print title, product_code, product_label, layerName, defaultStyle, fromdate, todate, prj + + metadata = { + "title": { + language: title + }, + "product_code": product_code, + "product_label": product_label, + "layerName": layerName, + "defaultStyle": defaultStyle, + "map_projection_code": map_projection_code, + "is_raster": True, + } + + # dates + if fromdate is not None: + metadata["from_date"] = fromdate + metadata["to_date"] = todate + + return metadata + +# TODO: check prj +def check_prj(prj): + print "Check if prj is valid or is the name" + return True + diff --git a/geobricks_data_scripts/utils/rename.py b/geobricks_data_scripts/utils/rename.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/utils/rename.py @@ -0,0 +1 @@ +__author__ = 'vortex' From 025eb7b69a0c524590bb735d392ca4862915c44b Mon Sep 17 00:00:00 2001 From: vortex Date: Fri, 23 Jan 2015 16:44:43 +0100 Subject: [PATCH 06/39] dev earthstat/ghg process layers --- geobricks_data_scripts/dev/config/config.py | 4 +- .../data/earthstat/processing/move_files.py | 24 +- .../data/earthstat/publish/publish_harvest.py | 3 +- .../delete/data_manager_delete_all_example.py | 2 +- .../dev/data/ghg/delete/delete_a_layer.py | 8 +- .../data/ghg/process/process_burned_areas.py | 215 +++++++++++++++++- .../dev/data/ghg/publish/publish_harvest.py | 56 ++++- 7 files changed, 298 insertions(+), 14 deletions(-) diff --git a/geobricks_data_scripts/dev/config/config.py b/geobricks_data_scripts/dev/config/config.py index aa56fde..900ef04 100644 --- a/geobricks_data_scripts/dev/config/config.py +++ b/geobricks_data_scripts/dev/config/config.py @@ -43,8 +43,8 @@ "geoserver": { "geoserver_master": "http://localhost:9090/geoserver/rest", "geoserver_slaves": [], - "username": "fenix", - "password": "Fenix2014", + "username": "admin", + "password": "geoserver", }, # Metadata diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py b/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py index 22d574d..89f5ec4 100644 --- a/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py +++ b/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py @@ -1 +1,23 @@ -__author__ = 'vortex' +import os +import glob +import shutil +from geobricks_common.core.filesystem import get_filename + +def copy_files(src_folder, file_type="geotiff"): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + # print folder + files = glob.glob(os.path.join(folder, "*." + file_type)) + for f in files: + filename = get_filename(f) + dst_file = os.path.join(src_folder, filename + "." + file_type) + print f + print dst_file + shutil.copyfile(f, dst_file) + + + + +# copy_files("/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/storage/earthstat_crop_area/") +#copy_files("/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/storage/earthstat_crop_yield/") \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py b/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py index 9f723d9..bdc0ddd 100644 --- a/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py +++ b/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py @@ -9,4 +9,5 @@ lang = "EN" workspace = "earthstat" -harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/", workspace) \ No newline at end of file +harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area/", workspace) +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/", workspace) \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py index b58c736..934a485 100644 --- a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py @@ -6,7 +6,7 @@ for layer in layers: try: print layer["uid"] - #data_manager.delete(layer["uid"]) + data_manager.delete(layer["uid"]) except Exception, e: print e pass \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py b/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py index f805fee..f9c5967 100644 --- a/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py +++ b/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py @@ -3,5 +3,9 @@ data_manager = get_data_manager() -data_manager.delete("ghg:dm_burning_savanna_2002_3857") -data_manager.delete("ghg:ch4_emissions_burning_otherforests_2004_3857") +# data_manager.delete("ghg:dm_burning_savanna_2002_3857") +# data_manager.delete("ghg:ch4_emissions_burning_otherforests_2004_3857") + +data_manager.delete("test:jrc_climate_zone_025deg_3857") +# data_manager.delete("test:cultivation_organic_soils_-_croplands_3857") +# data_manager.delete("cultivation_organic_soils_-_croplands_4326") \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py b/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py index 22d574d..0f885f2 100644 --- a/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py +++ b/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py @@ -1 +1,214 @@ -__author__ = 'vortex' +import glob +import os +import shutil +from geobricks_common.core.log import logger +from geobricks_common.core.filesystem import get_filename +from geobricks_processing.core.processing_core import process_obj + +log = logger(__file__) + +output_extension = "geotiff" + + +process_obj_3857 = [ + { + # "source_path": [source_path], + # "output_path": output_path, + # "output_file_name": "burundi_maize_area_4326.tif", + "band": 1, + "process": [ + { + "gdal_translate": { + "opt": { + "-of": "GTiff", + "-a_nodata": "0", + "-a_srs": "EPSG:3857", + "-co": "'TILED=YES'", + "-co": "'COMPRESS=DEFLATE'", + } + } + }, + ] + }, + { + "band": 1, + "process": [ + { + "gdaladdo": { + "parameters": { + # "--config": "BIGTIFF_OVERVIEW IF_NEEDED" + }, + "overviews_levels": "2 4 8 16" + } + } + ] + } +] + + +process_obj_4326 = [ + { + "band": 1, + "process": [ + { + "gdal_translate": { + "opt": { + "-of": "GTiff", + "-a_nodata": "0", + "-a_srs": "EPSG:4326", + "-co": "'COMPRESS=DEFLATE'", + } + } + } + ] + } +] + + +def process(src_folder, dest_folder, file_type): + # folders = glob.glob(os.path.join(src_folder, "*")) + # for folder in folders: + # if os.path.isdir(folder): + # dest_folder = create_output_folder(folder, folder_type, overwrite) + output_path = create_output_folder(dest_folder) + files = glob.glob(os.path.join(src_folder, "*." + file_type)) + for f in files: + filename = get_filename(f) + if "3857" in f: + prj = "3857" + else: + prj = "4326" + + # TODO: switch between processes + if prj == "3857": + output_file_name = filename + ".geotiff" + print "Processing:", output_file_name + # TODO: worksaround for the various sourcepath e dstpath + process_obj_3857[0]["source_path"] = [f] + process_obj_3857[0]["output_path"] = output_path + process_obj_3857[0]["output_file_name"] = output_file_name + for p in process_obj_3857: + p["source_path"] = p["source_path"] if "source_path" in p else result + result = process_obj(p) + elif prj == "4326": + output_file_name = filename + "_" + prj + ".geotiff" + print "Processing:", output_file_name + process_obj_4326[0]["source_path"] = [f] + process_obj_4326[0]["output_path"] = output_path + process_obj_4326[0]["output_file_name"] = output_file_name + for p in process_obj_4326: + p["source_path"] = p["source_path"] if "source_path" in p else result + result = process_obj(p) + + +def create_output_folder(folder): + # if os.path.exists(folder): + # shutil.rmtree(folder) + + if not os.path.exists(folder): + os.makedirs(folder) + + return folder + + +def process_burned_areas_foolders(src_folder, dest_folder, file_type): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + print folder + dest = os.path.join(folder, dest_folder) + process(folder, dest, file_type) + + +def rename_folders(src_folder): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + os.rename(folder, folder.replace(" ", "_")) + + +def rename_burned_areas_output_files(src_folder, dest_folder, file_type="geotiff"): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + output_folder = os.path.join(folder, dest_folder) + files = glob.glob(os.path.join(output_folder, "*." + file_type)) + for f in files: + print f + filename = get_filename(f) + renamed_filename = burned_areas_switch(filename).replace(" ", "_") + prj = filename.split("_")[len(filename.split("_"))-1] + year = filename.split("_")[len(filename.split("_"))-2] + renamed_filename += "_" + year + "_" + prj + "." + file_type + os.rename(f, os.path.join(output_folder, renamed_filename)) + + +def burned_areas_switch(filename): + filename = filename.lower() + if "peatlands" in filename: + return "Burned_Areas_-_Organic_soils" + if "AllForestsMinus".lower() in filename: + return "Burned_Areas_-_Other forest" + if "humidtropicalforests".lower() in filename: + return "Burned Areas_-_Humid Tropical Forest" + + burned_areas_lc = "Burned Areas_-_" + if "lc_1_" in filename: + return burned_areas_lc + "Evergreen Needleleaf forest" + if "lc_2_" in filename: + return burned_areas_lc + "Evergreen Broadleaf forest" + if "lc_3_" in filename: + return burned_areas_lc + "Deciduous Needleleaf forest" + if "lc_4_" in filename: + return burned_areas_lc + "Deciduous Broadleaf forest" + if "lc_5_" in filename: + return burned_areas_lc + "Mixed forest" + if "lc_6_" in filename: + return burned_areas_lc + "Closed shrubland" + if "lc_7_" in filename: + return burned_areas_lc + "Open shrubland" + if "lc_8_" in filename: + return burned_areas_lc + "Woody savanna" + if "lc_9_" in filename: + return burned_areas_lc + "Savanna" + if "lc_10_" in filename: + return burned_areas_lc + "Grassland" + if "lc_12_" in filename: + return burned_areas_lc + "Croplands" + if "lc_13_" in filename: + print filename + return burned_areas_lc + "Urban and built-up" + if "lc_16_" in filename: + return burned_areas_lc + "Barren or sparsely vegetated" + if "lc_17_" in filename: + return burned_areas_lc + "Unclassified" + + + +# load scripts +if __name__ == '__main__': + src_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GFED4_BURNEDAREAS_BY_LANDCOVER/" + dest_folder = "output" + + #rename_folders(src_folder) + #process_burned_areas_foolders(src_folder, dest_folder, "tif") + #process_burned_areas_foolders(src_folder, dest_folder, "tiff") + + #rename output files + #rename_burned_areas_output_files(src_folder, dest_folder) + + + #process(src_folder, dest_folder, "geoserver", "3857") + #process(src_folder, dest_folder, "storage", "4326") + + #move_storage_folder(os.path.join(dest_folder, "earthstat_crop_area/storage/")) + #move_storage_folder(os.path.join(dest_folder, "earthstat_crop_yield/storage/")) + + # N.B. the final move has been made by hand + # earthstat/earthstat_processeddata/ + # earthstat/earthstat_processeddata/geoserver/ + # earthstat/earthstat_processeddata/geoserver/earthstat_crop_area/ + # earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/ + # earthstat/earthstat_processeddata/storage/earthstat_crop_area/ + # earthstat/earthstat_processeddata/storage/earthstat_crop_area/abacaarea_4326/ + # etc... \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py b/geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py index 93440fd..3d04885 100644 --- a/geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py +++ b/geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py @@ -1,5 +1,6 @@ import glob import os +from geobricks_common.core.filesystem import get_filename from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style @@ -32,10 +33,13 @@ def get_maghg_defaultstyle_name(filename): # harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/Cultivation_organic_soils_croplands/Cultivation_organic_soils_-_croplands/", workspace) # harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GriddedLivestock/gridded_livestock_of_the_world_v_201/", workspace) # harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/JRC_CLIMATE_ZONE/JRC_climate_zone/", workspace) -# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/glc2000/glc2000_v1/", workspace) +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GLC2000/glc2000_v1/", workspace) # harvest_folder(data_manager, "/home/vortex/Deskt/op/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GEZ/global_ecological_zones_GEZ/", workspace) +def harvest_burned_areas(src_folder, output_folder="output", ): + print "TODO:" + # GHG-MAG # 4326 def harvest_maghg_4326(): @@ -74,14 +78,54 @@ def harvest_maghg_3857_updated_defaultstyle(): print e - - - +# def create_storage_json_metadata(src_folder, code, product_codelist_id="layers_products"): +# for glob=glob +# +# +# def create_product_json(code, product_codelist_id="layers_products"): +# metadata = { +# "meContent" : { +# "seCoverage" : { +# "coverageSectors" : { +# "version" : "1.0", +# "codes" : [ { +# "code" : code +# } ], +# "idCodeList" : product_codelist_id +# } +# } +# } +# } +# return metadata + + + +# DIRTY PRODUCT FIX +# def update_maghg_4326_storage_product(folder, workspace): +# # there is an issue with the product code of the 4326 layers (they need to be update with the 3857 relative one) +# folders = glob.glob(os.path.join(folder, "*")) +# for folder in folders: +# if os.path.isdir(folder): +# files = glob.glob(os.path.join(folder, "*.geotiff")) +# for f in files: +# uid = workspace + ":" +# uid += "_".join(get_filename(f).lower().split("_")[:-1]) + "_3857" +# print uid +# try: +# metadata = data_manager.metadata_manager.get_by_uid(uid) +# print metadata["dsd"]["contextExtension"]["distribution"][0]["uid"] +# except Exception, e: +# print "ERROR: ", e + + #data_manager.metadata_manager.update_metadata() #harvest_maghg_4326() -harvest_maghg_3857() -harvest_maghg_3857_updated_defaultstyle() +#harvest_maghg_3857() +#harvest_maghg_3857_updated_defaultstyle() + +# maghg_4326_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/storage" +# update_maghg_4326_storage_product(maghg_4326_folder, workspace) From 6c2cb0c2c629f2ad50190af1e2d80a5515f31a60 Mon Sep 17 00:00:00 2001 From: vortex Date: Fri, 23 Jan 2015 16:44:55 +0100 Subject: [PATCH 07/39] harvest update --- .../utils/harvest/harvest_rasters.py | 5 ++-- .../utils/harvest/publish_harvest.py | 25 ++++++++++++------- .../utils/metadata/metadata.py | 3 +++ geobricks_data_scripts/utils/rename.py | 17 ++++++++++++- 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/geobricks_data_scripts/utils/harvest/harvest_rasters.py b/geobricks_data_scripts/utils/harvest/harvest_rasters.py index 6e9044f..9d2d431 100644 --- a/geobricks_data_scripts/utils/harvest/harvest_rasters.py +++ b/geobricks_data_scripts/utils/harvest/harvest_rasters.py @@ -18,13 +18,13 @@ supported_file = ["tif", "tiff", "geotiff", "TIFF", "TIF", "GEOTIFF"] -def harvest_raster_folder(path): +def harvest_raster_folder(path, publish_metadata=True, publish_geoserver=True, publish_storage=True): # read files metadatas = [] files = glob.glob(os.path.join(path, "*")) # getting product code from the folder name - product_code = os.path.basename(os.path.normpath(path)) + product_code = sanitize_name(os.path.basename(os.path.normpath(path))) for f in files: if os.path.isfile(f): @@ -53,6 +53,7 @@ def harvest_raster_folder(path): metadata = dict_merge(metadata, d) + if metadata: metadata["path"] = f metadatas.append(metadata) diff --git a/geobricks_data_scripts/utils/harvest/publish_harvest.py b/geobricks_data_scripts/utils/harvest/publish_harvest.py index c9911bd..ef9c1fc 100644 --- a/geobricks_data_scripts/utils/harvest/publish_harvest.py +++ b/geobricks_data_scripts/utils/harvest/publish_harvest.py @@ -5,32 +5,36 @@ # TODO: apply publish on storage and geoserver -def harvest_folder(data_manager, folder, workspace=None, publish_on_geoserver=True, publish_on_storage=True, update_links=True): +def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, publish_on_storage=True, update_links=True, metadata_json=None): metadatas = harvest_raster_folder(folder) print metadatas # harvest 3857 for publication for metadata in metadatas: - print metadata + log.info(metadata) # check epsg if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": if "workspace" not in metadata["dsd"]: metadata["dsd"]["workspace"] = workspace if publish_on_geoserver: - publish_geoserver(data_manager, metadata["path"], metadata) + metadata = publish_geoserver(data_manager, metadata["path"], metadata) + log.info(metadata) + # harvest 4326 (or others) to storage # it's done after the 3857 it's sure the metadata will be there for metadata in metadatas: + log.info(metadata) if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": if publish_on_storage: - publish_storage(data_manager, metadata["path"], metadata, workspace) - + metadata = publish_storage(data_manager, metadata["path"], metadata, workspace) + log.info(metadata) + log.info(metadatas) if update_links: # link 3857 layer to 4326 distribution layer for metadata in metadatas: - print metadata + log.info(metadata) # check epsg if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": update_dsd_layer_to_distribution_layer(data_manager, metadata) @@ -38,7 +42,7 @@ def harvest_folder(data_manager, folder, workspace=None, publish_on_geoserver=Tr # link 4326 layer to 3857 visualization layer for metadata in metadatas: - print metadata + log.info(metadata) # check epsg if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace) @@ -69,17 +73,19 @@ def update_folder_style(data_manager, folder, style=None): def update_dsd_layer_to_distribution_layer(data_manager, metadata): + log.info(metadata) update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], "4326", "distribution") def update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace): + log.info(metadata) update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], "3857", "visualization", workspace) def publish_geoserver(data_manager, path, metadata): log.info("publish on geoserver") try: - data_manager.publish_coveragestore(path, metadata) + return data_manager.publish_coveragestore(path, metadata) except Exception, e: log.error(e) @@ -87,7 +93,7 @@ def publish_geoserver(data_manager, path, metadata): def publish_storage(data_manager, path, metadata, workspace=None, uid_distribution=True): log.info("publish on storage") try: - data_manager.publish_coveragestore_storage(path, metadata, False, False, True) + return data_manager.publish_coveragestore_storage(path, metadata, False, False, True) except Exception, e: log.error(e) @@ -131,3 +137,4 @@ def update_dsd_layer_with_layer_dist_or_vis(data_manager, uid, epsg_code="4326", # log.error("No metadata found for ", str(layer_uid)) log.error(e) + diff --git a/geobricks_data_scripts/utils/metadata/metadata.py b/geobricks_data_scripts/utils/metadata/metadata.py index 945c343..7f89996 100644 --- a/geobricks_data_scripts/utils/metadata/metadata.py +++ b/geobricks_data_scripts/utils/metadata/metadata.py @@ -1,9 +1,12 @@ import calendar import datetime +from geobricks_common.core.log import logger +log = logger(__file__) def create_metadata(metadata): + log.info(metadata) product_codelist_id = "layers_products" if "product_codelist_id" not in metadata else metadata["layers_products"] product_codelist_version = "1.0" if "product_codelist_version" not in metadata else metadata["product_codelist_version"] diff --git a/geobricks_data_scripts/utils/rename.py b/geobricks_data_scripts/utils/rename.py index 22d574d..8b85c23 100644 --- a/geobricks_data_scripts/utils/rename.py +++ b/geobricks_data_scripts/utils/rename.py @@ -1 +1,16 @@ -__author__ = 'vortex' +import os +import glob +from geobricks_common.core.filesystem import get_filename + +def rename_files(src_folder, from_file_type="tif", to_file_type="geotiff"): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + files = glob.glob(os.path.join(folder, "*." + from_file_type)) + for f in files: + filename = get_filename(f) + output_path = os.path.join(folder, filename + "." + to_file_type) + os.rename(f, output_path) + + +# rename_files("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/geoserver/") \ No newline at end of file From bfe598a5c008fe62f85842a25c848c8a1e5c523d Mon Sep 17 00:00:00 2001 From: vortex Date: Fri, 23 Jan 2015 16:45:06 +0100 Subject: [PATCH 08/39] updated demo files --- .../data/publish/publish_3857_13_novemeber.py | 22 +++++++++---------- ...ghg_create_folders_move_files_rename_it.py | 17 +++++++++++++- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/geobricks_data_scripts/demo/ghg/data/publish/publish_3857_13_novemeber.py b/geobricks_data_scripts/demo/ghg/data/publish/publish_3857_13_novemeber.py index 9a796ee..1853068 100644 --- a/geobricks_data_scripts/demo/ghg/data/publish/publish_3857_13_novemeber.py +++ b/geobricks_data_scripts/demo/ghg/data/publish/publish_3857_13_novemeber.py @@ -75,7 +75,7 @@ def publish_data_GriddedLivestock(input_folder): metadata_def = create_metadata(title, product, date, sldname) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e #manager.publish_coverage(input_file, metadata_def, False, False) @@ -97,7 +97,7 @@ def publish_data_Climate_Zones_processed(input_folder): metadata_def = create_metadata(title, product, date, sldname, map_projection_code) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e #manager.publish_coverage(input_file, metadata_def, False, False) @@ -119,7 +119,7 @@ def publish_data_modis_landcover(input_folder): metadata_def = create_metadata(title, product, date, sldname, map_projection_code) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e #manager.publish_coverage(input_file, metadata_def, False, False) @@ -149,7 +149,7 @@ def publish_burnerdareas(): metadata_def = create_metadata(title, product, date, sldname) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e #manager.publish_coverage(input_file, metadata_def, False, False) @@ -163,7 +163,7 @@ def publish_burnerdareas(): metadata_def = create_metadata(title, product, date, sldname) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e #manager.publish_coverage(input_file, metadata_def, False, False) @@ -179,7 +179,7 @@ def publish_burnerdareas(): metadata_def = create_metadata(title, product, date, sldname) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e @@ -245,7 +245,7 @@ def publish_area_of_histosols(path): metadata_def = create_metadata(title, product, date, sldname) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e @@ -260,7 +260,7 @@ def publish_gez(path): metadata_def = create_metadata(title, product, date, sldname) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e @@ -275,7 +275,7 @@ def publish_ghg_glc2000_v1_1(path): metadata_def = create_metadata(title, product, date, sldname) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e @@ -292,7 +292,7 @@ def publish_cultivation_organic_soils_croplands(path): metadata_def = create_metadata(title, product, date, sldname) try: print json.dumps(metadata_def) - data_manager.publish_coveragestore(input_file, metadata_def, False, False, True) + data_manager.publish_coveragestore(input_file, metadata_def, False, True, False) except Exception, e: print e @@ -310,7 +310,7 @@ def sanitize_name(name): # publish_data_GriddedLivestock("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/GriddedLivestock/to_publish_3857/") # publish_data_Climate_Zones_processed("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Climate_Zones_processed/to_publish/") -publish_data_modis_landcover("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MCD12Q1/processed_2009/3857/") +# publish_data_modis_landcover("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MCD12Q1/processed_2009/3857/") # publish_burnerdareas() # publish_area_of_histosols("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/HWSD/3857/") # publish_gez("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/gez_raster/3857/") diff --git a/geobricks_data_scripts/demo/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py b/geobricks_data_scripts/demo/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py index dfc5d26..bdfc271 100644 --- a/geobricks_data_scripts/demo/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py +++ b/geobricks_data_scripts/demo/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py @@ -45,4 +45,19 @@ def create_folder(path, filename): -process_files() \ No newline at end of file + + +def move_to_another_folder(src_folder, file_type="geotiff"): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + files = glob.glob(os.path.join(folder, "*." + file_type)) + for f in files: + print f, src_folder + filename = get_filename(f) + output_path = os.path.join(src_folder, filename+ "." + file_type) + shutil.copyfile(f, output_path) + + +#process_files() +move_to_another_folder("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/storage/") \ No newline at end of file From 7b3f61e20797a4e56a7594c5dda760b20d25ea7a Mon Sep 17 00:00:00 2001 From: vortex Date: Fri, 23 Jan 2015 16:45:28 +0100 Subject: [PATCH 09/39] updated test env --- .../test/coding_system/csv/final.csv | 83 +++++-------------- .../csv/final_without_maghg_and_soil.csv | 81 ++++-------------- .../test/coding_system/publish_final_cs.py | 2 +- .../coding_system/update_mapProjections.py | 2 +- .../delete/data_manager_delete_all_example.py | 3 +- .../test/data/ghg/publish/publish_GHG_data.py | 10 +-- .../test/utils/data_manager_util.py | 2 +- 7 files changed, 46 insertions(+), 137 deletions(-) diff --git a/geobricks_data_scripts/test/coding_system/csv/final.csv b/geobricks_data_scripts/test/coding_system/csv/final.csv index 43b1ff4..90c32c6 100644 --- a/geobricks_data_scripts/test/coding_system/csv/final.csv +++ b/geobricks_data_scripts/test/coding_system/csv/final.csv @@ -1,68 +1,23 @@ -cultivation_organic_soils,Cultivation Organic Soils -gfed4_burned_areas_-_barren_or_sparsely_vegetated,GFED4 Burned Areas - Barren or sparsely vegetated -gfed4_burned_areas_-_closed_shrubland,GFED4 Burned Areas - Closed shrubland -gfed4_burned_areas_-_croplands,GFED4 Burned Areas - Croplands -gfed4_burned_areas_-_deciduous_broadleaf_forest,GFED4 Burned Areas - Deciduous Broadleaf forest -gfed4_burned_areas_-_deciduous_needleleaf_forest,GFED4 Burned Areas - Deciduous Needleleaf forest -gfed4_burned_areas_-_evergreen_broadleaf_forest,GFED4 Burned Areas - Evergreen Broadleaf forest -gfed4_burned_areas_-_evergreen_needleleaf_forest,GFED4 Burned Areas - Evergreen Needleleaf forest -gfed4_burned_areas_-_grassland,GFED4 Burned Areas - Grassland -gfed4_burned_areas_-_humid_tropical_forest,GFED4 Burned Areas - Humid Tropical Forest -gfed4_burned_areas_-_mixed_forest,GFED4 Burned Areas - Mixed forest -gfed4_burned_areas_-_open_shrubland,GFED4 Burned Areas - Open shrubland -gfed4_burned_areas_-_organic_soils,GFED4 Burned Areas - Organic soils -gfed4_burned_areas_-_other_forest,GFED4 Burned Areas - Other forest -gfed4_burned_areas_-_savanna,GFED4 Burned Areas - Savanna -gfed4_burned_areas_-_unclassified,GFED4 Burned Areas - Unclassified -gfed4_burned_areas_-_urban_and_built-up,GFED4 Burned Areas - Urban and built-up -gfed4_burned_areas_-_woody_savanna,GFED4 Burned Areas - Woody savanna +cultivation_organic_soils_-_croplands,Cultivation Organic Soils - Croplands +burned_areas_-_barren_or_sparsely_vegetated,Burned Areas - Barren or sparsely vegetated +burned_areas_-_closed_shrubland,Burned Areas - Closed shrubland +burned_areas_-_croplands,Burned Areas - Croplands +burned_areas_-_deciduous_broadleaf_forest,Burned Areas - Deciduous Broadleaf forest +burned_areas_-_deciduous_needleleaf_forest,Burned Areas - Deciduous Needleleaf forest +burned_areas_-_evergreen_broadleaf_forest,Burned Areas - Evergreen Broadleaf forest +burned_areas_-_evergreen_needleleaf_forest,Burned Areas - Evergreen Needleleaf forest +burned_areas_-_grassland,Burned Areas - Grassland +burned_areas_-_humid_tropical_forest,Burned Areas - Humid Tropical Forest +burned_areas_-_mixed_forest,Burned Areas - Mixed forest +burned_areas_-_open_shrubland,Burned Areas - Open shrubland +burned_areas_-_organic_soils,Burned Areas - Organic soils +burned_areas_-_other_forest,Burned Areas - Other forest +burned_areas_-_savanna,Burned Areas - Savanna +burned_areas_-_unclassified,Burned Areas - Unclassified +burned_areas_-_urban_and_built-up,Burned Areas - Urban and built-up +burned_areas_-_woody_savanna,Burned Areas - Woody savanna global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 jrc_climate_zone,JRC climate zone -modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD -dm_burning_grasslands,DM Burning Grasslands -ch4_emissions_burning_woodysavanna,CH4 Emissions Burning WoodySavanna -n2o_emissions_burning_grasslands,N2O Emissions Burning Grasslands -n2o_emissions_burning_woodysavanna,N2O Emissions Burning WoodySavanna -co2_emissions_burning_closedshrublands,CO2 Emissions Burning ClosedShrublands -co2_emissions_burning_openshrublands,CO2 Emissions Burning OpenShrublands -co2_emissions_burning_humidtropicalforests,CO2 Emissions Burning HumidTropicalForests -dm_burning_openshrublands,DM Burning OpenShrublands -n2o_emissions_burning_otherforests,N2O Emissions Burning OtherForests -n2o_emissions_burning_humidtropicalforests,N2O Emissions Burning HumidTropicalForests -n2o_emissions_burning_savanna,N2O Emissions Burning Savanna -ch4_emissions_burning_humidtropicalforests,CH4 Emissions Burning HumidTropicalForests -ch4_emissions_burning_peatlands,CH4 Emissions Burning Peatlands -n2o_emissions_burning_peatlands,N2O Emissions Burning Peatlands -ch4_emissions_burning_closedshrublands,CH4 Emissions Burning ClosedShrublands -co2_emissions_burning_grasslands,CO2 Emissions Burning Grasslands -ch4_emissions_burning_grasslands,CH4 Emissions Burning Grasslands -openshrublands_fuelbiomass,OpenShrublands FuelBiomass -co2_emissions_burning_woodysavanna,CO2 Emissions Burning WoodySavanna -ch4_emissions_burning_otherforests,CH4 Emissions Burning OtherForests -dm_burning_closedshrublands,DM Burning ClosedShrublands -dm_burning_woodysavanna,DM Burning WoodySavanna -ch4_emissions_burning_savanna,CH4 Emissions Burning Savanna -co2_emissions_burning_otherforests,CO2 Emissions Burning OtherForests -n2o_emissions_burning_closedshrublands,N2O Emissions Burning ClosedShrublands -co2_emissions_burning_peatlands,CO2 Emissions Burning Peatlands -co2_emissions_burning_savanna,CO2 Emissions Burning Savanna -dm_burning_peatlands,DM Burning Peatlands -dm_burning_humidtropicalforests,DM Burning HumidTropicalForests -savanna_fuelbiomass,Savanna FuelBiomass -n2o_emissions_burning_openshrublands,N2O Emissions Burning OpenShrublands -tropical_forests_ch4_ef,Tropical Forests CH4 EF -dm_burning_savanna,DM Burning Savanna -ch4_emissions_burning_openshrublands,CH4 Emissions Burning OpenShrublands -dm_burning_otherforests,DM Burning OtherForests -tropical_forests_fuelbiomass,Tropical Forests FuelBiomass -woodysavanna_fuelbiomass,WoodySavanna FuelBiomass -grasslands_fuelbiomass,Grasslands FuelBiomass -nontropical_forests_fuelbiomass,NonTropical Forests FuelBiomass -tropical_forests_co2_ef,Tropical Forests CO2 EF -extratropical_forests_n2o_ef,ExtraTropical Forests N2O EF -closedshrublands_fuelbiomass,ClosedShrublands FuelBiomass -tropical_forests_n2o_ef,Tropical Forests N2O EF -extratropical_forests_co2_ef,ExtraTropical Forests CO2 EF -extratropical_forests_ch4_ef,ExtraTropical Forests CH4 EF +modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD \ No newline at end of file diff --git a/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv b/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv index 43b1ff4..403530d 100644 --- a/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv +++ b/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv @@ -1,68 +1,23 @@ -cultivation_organic_soils,Cultivation Organic Soils -gfed4_burned_areas_-_barren_or_sparsely_vegetated,GFED4 Burned Areas - Barren or sparsely vegetated -gfed4_burned_areas_-_closed_shrubland,GFED4 Burned Areas - Closed shrubland -gfed4_burned_areas_-_croplands,GFED4 Burned Areas - Croplands -gfed4_burned_areas_-_deciduous_broadleaf_forest,GFED4 Burned Areas - Deciduous Broadleaf forest -gfed4_burned_areas_-_deciduous_needleleaf_forest,GFED4 Burned Areas - Deciduous Needleleaf forest -gfed4_burned_areas_-_evergreen_broadleaf_forest,GFED4 Burned Areas - Evergreen Broadleaf forest -gfed4_burned_areas_-_evergreen_needleleaf_forest,GFED4 Burned Areas - Evergreen Needleleaf forest -gfed4_burned_areas_-_grassland,GFED4 Burned Areas - Grassland -gfed4_burned_areas_-_humid_tropical_forest,GFED4 Burned Areas - Humid Tropical Forest -gfed4_burned_areas_-_mixed_forest,GFED4 Burned Areas - Mixed forest -gfed4_burned_areas_-_open_shrubland,GFED4 Burned Areas - Open shrubland -gfed4_burned_areas_-_organic_soils,GFED4 Burned Areas - Organic soils -gfed4_burned_areas_-_other_forest,GFED4 Burned Areas - Other forest -gfed4_burned_areas_-_savanna,GFED4 Burned Areas - Savanna -gfed4_burned_areas_-_unclassified,GFED4 Burned Areas - Unclassified -gfed4_burned_areas_-_urban_and_built-up,GFED4 Burned Areas - Urban and built-up -gfed4_burned_areas_-_woody_savanna,GFED4 Burned Areas - Woody savanna +cultivation_organic_soils_-_croplands,Cultivation Organic Soils - Croplands +burned_areas_-_barren_or_sparsely_vegetated,Burned Areas - Barren or sparsely vegetated +burned_areas_-_closed_shrubland,Burned Areas - Closed shrubland +burned_areas_-_croplands,Burned Areas - Croplands +burned_areas_-_deciduous_broadleaf_forest,Burned Areas - Deciduous Broadleaf forest +burned_areas_-_deciduous_needleleaf_forest,Burned Areas - Deciduous Needleleaf forest +burned_areas_-_evergreen_broadleaf_forest,Burned Areas - Evergreen Broadleaf forest +burned_areas_-_evergreen_needleleaf_forest,Burned Areas - Evergreen Needleleaf forest +burned_areas_-_grassland,Burned Areas - Grassland +burned_areas_-_humid_tropical_forest,Burned Areas - Humid Tropical Forest +burned_areas_-_mixed_forest,Burned Areas - Mixed forest +burned_areas_-_open_shrubland,Burned Areas - Open shrubland +burned_areas_-_organic_soils,Burned Areas - Organic soils +burned_areas_-_other_forest,Burned Areas - Other forest +burned_areas_-_savanna,Burned Areas - Savanna +burned_areas_-_unclassified,Burned Areas - Unclassified +burned_areas_-_urban_and_built-up,Burned Areas - Urban and built-up +burned_areas_-_woody_savanna,Burned Areas - Woody savanna global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 jrc_climate_zone,JRC climate zone modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD -dm_burning_grasslands,DM Burning Grasslands -ch4_emissions_burning_woodysavanna,CH4 Emissions Burning WoodySavanna -n2o_emissions_burning_grasslands,N2O Emissions Burning Grasslands -n2o_emissions_burning_woodysavanna,N2O Emissions Burning WoodySavanna -co2_emissions_burning_closedshrublands,CO2 Emissions Burning ClosedShrublands -co2_emissions_burning_openshrublands,CO2 Emissions Burning OpenShrublands -co2_emissions_burning_humidtropicalforests,CO2 Emissions Burning HumidTropicalForests -dm_burning_openshrublands,DM Burning OpenShrublands -n2o_emissions_burning_otherforests,N2O Emissions Burning OtherForests -n2o_emissions_burning_humidtropicalforests,N2O Emissions Burning HumidTropicalForests -n2o_emissions_burning_savanna,N2O Emissions Burning Savanna -ch4_emissions_burning_humidtropicalforests,CH4 Emissions Burning HumidTropicalForests -ch4_emissions_burning_peatlands,CH4 Emissions Burning Peatlands -n2o_emissions_burning_peatlands,N2O Emissions Burning Peatlands -ch4_emissions_burning_closedshrublands,CH4 Emissions Burning ClosedShrublands -co2_emissions_burning_grasslands,CO2 Emissions Burning Grasslands -ch4_emissions_burning_grasslands,CH4 Emissions Burning Grasslands -openshrublands_fuelbiomass,OpenShrublands FuelBiomass -co2_emissions_burning_woodysavanna,CO2 Emissions Burning WoodySavanna -ch4_emissions_burning_otherforests,CH4 Emissions Burning OtherForests -dm_burning_closedshrublands,DM Burning ClosedShrublands -dm_burning_woodysavanna,DM Burning WoodySavanna -ch4_emissions_burning_savanna,CH4 Emissions Burning Savanna -co2_emissions_burning_otherforests,CO2 Emissions Burning OtherForests -n2o_emissions_burning_closedshrublands,N2O Emissions Burning ClosedShrublands -co2_emissions_burning_peatlands,CO2 Emissions Burning Peatlands -co2_emissions_burning_savanna,CO2 Emissions Burning Savanna -dm_burning_peatlands,DM Burning Peatlands -dm_burning_humidtropicalforests,DM Burning HumidTropicalForests -savanna_fuelbiomass,Savanna FuelBiomass -n2o_emissions_burning_openshrublands,N2O Emissions Burning OpenShrublands -tropical_forests_ch4_ef,Tropical Forests CH4 EF -dm_burning_savanna,DM Burning Savanna -ch4_emissions_burning_openshrublands,CH4 Emissions Burning OpenShrublands -dm_burning_otherforests,DM Burning OtherForests -tropical_forests_fuelbiomass,Tropical Forests FuelBiomass -woodysavanna_fuelbiomass,WoodySavanna FuelBiomass -grasslands_fuelbiomass,Grasslands FuelBiomass -nontropical_forests_fuelbiomass,NonTropical Forests FuelBiomass -tropical_forests_co2_ef,Tropical Forests CO2 EF -extratropical_forests_n2o_ef,ExtraTropical Forests N2O EF -closedshrublands_fuelbiomass,ClosedShrublands FuelBiomass -tropical_forests_n2o_ef,Tropical Forests N2O EF -extratropical_forests_co2_ef,ExtraTropical Forests CO2 EF -extratropical_forests_ch4_ef,ExtraTropical Forests CH4 EF diff --git a/geobricks_data_scripts/test/coding_system/publish_final_cs.py b/geobricks_data_scripts/test/coding_system/publish_final_cs.py index 0d975e3..8852627 100644 --- a/geobricks_data_scripts/test/coding_system/publish_final_cs.py +++ b/geobricks_data_scripts/test/coding_system/publish_final_cs.py @@ -2,7 +2,7 @@ import json from geobricks_metadata_manager.core.metadata_manager_d3s_core import MetadataManager -from geobricks_data_scripts.demo.config.config import settings +from geobricks_data_scripts.test.config.config import settings metadata_manager = MetadataManager(settings) diff --git a/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py b/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py index c92b6cf..e13438a 100644 --- a/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py +++ b/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py @@ -1,4 +1,4 @@ -from geobricks_data_scripts.demo.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager data_manager = get_data_manager() diff --git a/geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py index 934a485..80723bc 100644 --- a/geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py @@ -1,5 +1,4 @@ - -from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager data_manager = get_data_manager() layers = data_manager.get_all_layers() diff --git a/geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py index 7edb3fa..49b9b1c 100644 --- a/geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py +++ b/geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py @@ -1,14 +1,14 @@ import os import glob from geobricks_common.core.log import logger -from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager -from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style +from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder log = logger(__file__) data_manager = get_data_manager() -workspace = "test" +workspace = "ghg" def default_import(src_folder): @@ -45,8 +45,8 @@ def set_style_modis(): # Burned areas burned_areas(os.path.join(base_folder, "GFED4_BURNEDAREAS_BY_LANDCOVER")) - # GLC 2000 - default_import(os.path.join(base_folder, "Global_Land_Cover_2000_(GLC_2000)", "Global_Land_Cover_2000_(GLC_2000)")) + # GLC 2000 (TODO: set the style) + #default_import(os.path.join(base_folder, "Global_Land_Cover_2000_(GLC_2000)", "Global_Land_Cover_2000_(GLC_2000)")) # griddedlivestock default_import(os.path.join(base_folder, "GriddedLivestock", "gridded_livestock_of_the_world_v_201")) diff --git a/geobricks_data_scripts/test/utils/data_manager_util.py b/geobricks_data_scripts/test/utils/data_manager_util.py index 357578c..2acda6e 100644 --- a/geobricks_data_scripts/test/utils/data_manager_util.py +++ b/geobricks_data_scripts/test/utils/data_manager_util.py @@ -1,5 +1,5 @@ from geobricks_data_manager.core.data_manager_core import DataManager -from geobricks_data_scripts.dev.config.config import settings +from geobricks_data_scripts.test.config.config import settings # get the default data manager for demo db def get_data_manager(): From e38648de6c3e3fa841dfd813a0cc22598fd67422 Mon Sep 17 00:00:00 2001 From: vortex Date: Mon, 26 Jan 2015 15:54:32 +0100 Subject: [PATCH 10/39] added harvest to test section --- .../data/earthstat/publish/publish_harvest.py | 3 ++- .../data/earthstat/publish/publish_harvest.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 geobricks_data_scripts/test/data/earthstat/publish/publish_harvest.py diff --git a/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py b/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py index bdc0ddd..b18cd2d 100644 --- a/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py +++ b/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py @@ -9,5 +9,6 @@ lang = "EN" workspace = "earthstat" -harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area/", workspace) +metadatas = harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area/", workspace, False, False, False, None) +print metadatas # harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/", workspace) \ No newline at end of file diff --git a/geobricks_data_scripts/test/data/earthstat/publish/publish_harvest.py b/geobricks_data_scripts/test/data/earthstat/publish/publish_harvest.py new file mode 100644 index 0000000..55e0bfd --- /dev/null +++ b/geobricks_data_scripts/test/data/earthstat/publish/publish_harvest.py @@ -0,0 +1,17 @@ +import glob +import os +from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder + + +data_manager = get_data_manager() + +lang = "EN" +workspace = "earthstat" + + +metadatas = harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area/", workspace) +print metadatas +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/", workspace) + + From 0863735791b169e8894e99933d146d06dd3fc386 Mon Sep 17 00:00:00 2001 From: vortex Date: Mon, 26 Jan 2015 15:55:00 +0100 Subject: [PATCH 11/39] added harvest to test section --- .../test/data/earthstat/__init__.py | 1 + .../test/data/earthstat/delete/__init__.py | 1 + .../data/earthstat/delete/delete_all_earthstat.py | 12 ++++++++++++ .../test/data/earthstat/publish/__init__.py | 1 + 4 files changed, 15 insertions(+) create mode 100644 geobricks_data_scripts/test/data/earthstat/__init__.py create mode 100644 geobricks_data_scripts/test/data/earthstat/delete/__init__.py create mode 100644 geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py create mode 100644 geobricks_data_scripts/test/data/earthstat/publish/__init__.py diff --git a/geobricks_data_scripts/test/data/earthstat/__init__.py b/geobricks_data_scripts/test/data/earthstat/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/data/earthstat/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/data/earthstat/delete/__init__.py b/geobricks_data_scripts/test/data/earthstat/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/data/earthstat/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py new file mode 100644 index 0000000..4ad344c --- /dev/null +++ b/geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py @@ -0,0 +1,12 @@ + +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +for layer in layers: + try: + if "earthstat" in layer["dsd"]["workspace"]: + data_manager.delete(layer["uid"]) + except Exception, e: + print "Error", e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/test/data/earthstat/publish/__init__.py b/geobricks_data_scripts/test/data/earthstat/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/test/data/earthstat/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' From 32b3190255242ac42f13a7e9491378dd7ea572ef Mon Sep 17 00:00:00 2001 From: vortex Date: Thu, 29 Jan 2015 15:37:12 +0100 Subject: [PATCH 12/39] delete all test earthstat data --- .../test/data/earthstat/delete/delete_all_earthstat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py index 4ad344c..8afa556 100644 --- a/geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py +++ b/geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py @@ -1,5 +1,4 @@ - -from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager data_manager = get_data_manager() layers = data_manager.get_all_layers() From f2f3de5cc73207aa3f2befa51c3e6424ce0251b8 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Tue, 10 Feb 2015 18:44:11 +0100 Subject: [PATCH 13/39] improved earthstat data processing --- .../data/earthstat/processing/move_files.py | 3 +- .../processing/processing_earthstat_data.py | 32 ++++++++++++++++--- .../dev/data/harvestchoice/__init__.py | 1 + .../dev/data/harvestchoice/harvestchoice.py | 1 + 4 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 geobricks_data_scripts/dev/data/harvestchoice/__init__.py create mode 100644 geobricks_data_scripts/dev/data/harvestchoice/harvestchoice.py diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py b/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py index 89f5ec4..c8ee2a6 100644 --- a/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py +++ b/geobricks_data_scripts/dev/data/earthstat/processing/move_files.py @@ -3,6 +3,7 @@ import shutil from geobricks_common.core.filesystem import get_filename +# Copy files used for Storage (4326 Layers) to be compliant with the storage_structure def copy_files(src_folder, file_type="geotiff"): folders = glob.glob(os.path.join(src_folder, "*")) for folder in folders: @@ -20,4 +21,4 @@ def copy_files(src_folder, file_type="geotiff"): # copy_files("/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/storage/earthstat_crop_area/") -#copy_files("/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/storage/earthstat_crop_yield/") \ No newline at end of file +# copy_files("/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/storage/earthstat_crop_yield/") \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py b/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py index 09fb8b0..0971cdf 100644 --- a/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py +++ b/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py @@ -176,6 +176,16 @@ def move_storage_folder(base_folder, file_type="geotiff"): shutil.move(f, folder) +def move_file_to_publish(src_folder, to_publish_folder, file_type="geotiff"): + files = glob.glob(os.path.join(src_folder, "*." + file_type)) + print src_folder + for f in files: + print f + if os.path.isfile(f): + filename = get_filename(f) + shutil.copyfile(f, to_publish_folder + "/" + filename +"." + file_type) + + # load scripts @@ -184,17 +194,29 @@ def move_storage_folder(base_folder, file_type="geotiff"): dest_folder = "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/" - #process(src_folder, dest_folder, "geoserver", "3857") + # process(src_folder, dest_folder, "geoserver", "3857") #process(src_folder, dest_folder, "storage", "4326") - #move_storage_folder(os.path.join(dest_folder, "earthstat_crop_area/storage/")) - #move_storage_folder(os.path.join(dest_folder, "earthstat_crop_yield/storage/")) + # MOVE TO PUBLISH FOLDER + to_publish_folder_crop_area = "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area" + to_publish_folder_crop_yield = "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_yield" + #move_file_to_publish(dest_folder + "earthstat_crop_area/geoserver/", to_publish_folder_crop_area) + #move_file_to_publish(dest_folder + "earthstat_crop_area/storage/", to_publish_folder_crop_area) + #move_file_to_publish(dest_folder + "earthstat_crop_yield/geoserver/", to_publish_folder_crop_yield) + #move_file_to_publish(dest_folder + "earthstat_crop_yield/storage/", to_publish_folder_crop_yield) + + + # This is used to create Storage folders to move to the server (INTHEORY IT'S NOT NEEDED) + # move_storage_folder(os.path.join(dest_folder, "earthstat_crop_area/storage/")) + # move_storage_folder(os.path.join(dest_folder, "earthstat_crop_yield/storage/")) - # N.B. the final move has been made by hand + + # NOT USED ANYMORE: N.B. the final move has been made MANUALLY # earthstat/earthstat_processeddata/ # earthstat/earthstat_processeddata/geoserver/ # earthstat/earthstat_processeddata/geoserver/earthstat_crop_area/ # earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/ # earthstat/earthstat_processeddata/storage/earthstat_crop_area/ # earthstat/earthstat_processeddata/storage/earthstat_crop_area/abacaarea_4326/ - # etc... \ No newline at end of file + # etc... + diff --git a/geobricks_data_scripts/dev/data/harvestchoice/__init__.py b/geobricks_data_scripts/dev/data/harvestchoice/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/harvestchoice/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/harvestchoice/harvestchoice.py b/geobricks_data_scripts/dev/data/harvestchoice/harvestchoice.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/harvestchoice/harvestchoice.py @@ -0,0 +1 @@ +__author__ = 'vortex' From 6d9e13b98ce544fd5cd65480c817582da9529367 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Fri, 13 Feb 2015 11:15:39 +0100 Subject: [PATCH 14/39] refactoring demo structure --- geobricks_data_scripts/demo/config/config.py | 77 ------------------- .../demo/{common_data => fenix}/__init__.py | 0 .../check_consistency}/__init__.py | 0 .../check_consistency/check_consistency.py | 14 ++++ .../coding_system}/__init__.py | 0 .../coding_system/create_cs_csv.py | 0 .../coding_system/csv/cs_ghg.csv | 0 .../coding_system/csv/cs_maghg.csv | 0 .../demo/fenix/coding_system/csv/final.csv | 23 ++++++ .../csv/final_without_maghg_and_soil.csv | 23 ++++++ .../coding_system/publish_final_cs.py | 3 +- .../coding_system/publish_maghg_cs.py | 0 .../{ghg => fenix/common_data}/__init__.py | 0 .../common_data}/coding_system/__init__.py | 0 .../coding_system/update_mapProjections.py | 2 +- .../demo/fenix/config/.gitignore | 2 + .../demo/{ghg => fenix}/data/__init__.py | 0 .../data/earthstat}/__init__.py | 0 .../data/earthstat/delete}/__init__.py | 0 .../earthstat/delete/delete_all_earthstat.py | 11 +++ .../data/earthstat/publish}/__init__.py | 0 .../data/earthstat/publish/publish_harvest.py | 15 ++++ .../publish => fenix/data/ghg}/__init__.py | 0 .../data/ghg/delete}/__init__.py | 0 .../delete/data_manager_delete_all_example.py | 11 +++ .../data/ghg/publish}/__init__.py | 0 .../data/ghg/publish/publish_GHG_data.py | 63 +++++++++++++++ .../demo/{config => fenix/utils}/.gitignore | 0 .../process => fenix/utils}/__init__.py | 0 .../{ => fenix}/utils/data_manager_util.py | 2 +- .../demo/{utils => old_ghg}/.gitignore | 0 .../process/ghg_maghg => old_ghg}/__init__.py | 0 .../common_data}/__init__.py | 0 .../common_data/coding_system}/__init__.py | 0 .../coding_system/update_mapProjections.py | 33 ++++++++ .../demo/old_ghg/config/.gitignore | 2 + .../demo/{utils => old_ghg/ghg}/__init__.py | 0 .../old_ghg/ghg/coding_system/__init__.py | 1 + .../ghg/coding_system/create_cs_csv.py | 48 ++++++++++++ .../old_ghg/ghg/coding_system/csv/cs_ghg.csv | 25 ++++++ .../ghg/coding_system/csv/cs_maghg.csv | 45 +++++++++++ .../ghg/coding_system/csv/final.csv | 0 .../csv/final_without_maghg_and_soil.csv | 0 .../ghg/coding_system/publish_final_cs.py | 44 +++++++++++ .../ghg/coding_system/publish_maghg_cs.py | 26 +++++++ .../demo/old_ghg/ghg/data/__init__.py | 1 + .../demo/old_ghg/ghg/data/delete/__init__.py | 1 + .../demo/{ => old_ghg}/ghg/data/delete/ghg.py | 5 +- .../demo/old_ghg/ghg/data/process/__init__.py | 1 + .../ghg/data/process/ghg_maghg/__init__.py | 1 + .../ghg_rename_files_and_processing.py | 0 .../demo/old_ghg/ghg/data/publish/__init__.py | 1 + .../data/publish/publish_3857_13_novemeber.py | 3 +- .../ghg/data/publish/publish_3857_maghg.py | 5 +- .../demo/old_ghg/ghg/data/update/__init__.py | 1 + .../ghg/data/update/update_metadata.py | 2 +- .../demo/old_ghg/ghg/data_storage/__init__.py | 1 + .../ghg/data_storage/process/__init__.py | 1 + .../process/ghg_maghg/__init__.py | 1 + ...ghg_create_folders_move_files_rename_it.py | 0 .../data_storage/process/nov_13/__init__.py | 1 + .../ghg/data_storage/publish/__init__.py | 1 + .../publish/publish_4326_maghg.py | 4 +- .../demo/old_ghg/utils/.gitignore | 1 + .../demo/old_ghg/utils/__init__.py | 1 + .../demo/old_ghg/utils/data_manager_util.py | 6 ++ 66 files changed, 418 insertions(+), 90 deletions(-) delete mode 100644 geobricks_data_scripts/demo/config/config.py rename geobricks_data_scripts/demo/{common_data => fenix}/__init__.py (100%) rename geobricks_data_scripts/demo/{common_data/coding_system => fenix/check_consistency}/__init__.py (100%) create mode 100644 geobricks_data_scripts/demo/fenix/check_consistency/check_consistency.py rename geobricks_data_scripts/demo/{config => fenix/coding_system}/__init__.py (100%) rename geobricks_data_scripts/demo/{ghg => fenix}/coding_system/create_cs_csv.py (100%) rename geobricks_data_scripts/demo/{ghg => fenix}/coding_system/csv/cs_ghg.csv (100%) rename geobricks_data_scripts/demo/{ghg => fenix}/coding_system/csv/cs_maghg.csv (100%) create mode 100644 geobricks_data_scripts/demo/fenix/coding_system/csv/final.csv create mode 100644 geobricks_data_scripts/demo/fenix/coding_system/csv/final_without_maghg_and_soil.csv rename geobricks_data_scripts/demo/{ghg => fenix}/coding_system/publish_final_cs.py (91%) rename geobricks_data_scripts/demo/{ghg => fenix}/coding_system/publish_maghg_cs.py (100%) rename geobricks_data_scripts/demo/{ghg => fenix/common_data}/__init__.py (100%) rename geobricks_data_scripts/demo/{ghg => fenix/common_data}/coding_system/__init__.py (100%) rename geobricks_data_scripts/demo/{ => fenix}/common_data/coding_system/update_mapProjections.py (88%) create mode 100644 geobricks_data_scripts/demo/fenix/config/.gitignore rename geobricks_data_scripts/demo/{ghg => fenix}/data/__init__.py (100%) rename geobricks_data_scripts/demo/{ghg/data/delete => fenix/data/earthstat}/__init__.py (100%) rename geobricks_data_scripts/demo/{ghg/data/process => fenix/data/earthstat/delete}/__init__.py (100%) create mode 100644 geobricks_data_scripts/demo/fenix/data/earthstat/delete/delete_all_earthstat.py rename geobricks_data_scripts/demo/{ghg/data/process/ghg_maghg => fenix/data/earthstat/publish}/__init__.py (100%) create mode 100644 geobricks_data_scripts/demo/fenix/data/earthstat/publish/publish_harvest.py rename geobricks_data_scripts/demo/{ghg/data/publish => fenix/data/ghg}/__init__.py (100%) rename geobricks_data_scripts/demo/{ghg/data/update => fenix/data/ghg/delete}/__init__.py (100%) create mode 100644 geobricks_data_scripts/demo/fenix/data/ghg/delete/data_manager_delete_all_example.py rename geobricks_data_scripts/demo/{ghg/data_storage => fenix/data/ghg/publish}/__init__.py (100%) create mode 100644 geobricks_data_scripts/demo/fenix/data/ghg/publish/publish_GHG_data.py rename geobricks_data_scripts/demo/{config => fenix/utils}/.gitignore (100%) rename geobricks_data_scripts/demo/{ghg/data_storage/process => fenix/utils}/__init__.py (100%) rename geobricks_data_scripts/demo/{ => fenix}/utils/data_manager_util.py (71%) rename geobricks_data_scripts/demo/{utils => old_ghg}/.gitignore (100%) rename geobricks_data_scripts/demo/{ghg/data_storage/process/ghg_maghg => old_ghg}/__init__.py (100%) rename geobricks_data_scripts/demo/{ghg/data_storage/process/nov_13 => old_ghg/common_data}/__init__.py (100%) rename geobricks_data_scripts/demo/{ghg/data_storage/publish => old_ghg/common_data/coding_system}/__init__.py (100%) create mode 100644 geobricks_data_scripts/demo/old_ghg/common_data/coding_system/update_mapProjections.py create mode 100644 geobricks_data_scripts/demo/old_ghg/config/.gitignore rename geobricks_data_scripts/demo/{utils => old_ghg/ghg}/__init__.py (100%) create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/coding_system/__init__.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/coding_system/create_cs_csv.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/cs_ghg.csv create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/cs_maghg.csv rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/coding_system/csv/final.csv (100%) rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/coding_system/csv/final_without_maghg_and_soil.csv (100%) create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/coding_system/publish_final_cs.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/coding_system/publish_maghg_cs.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data/__init__.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data/delete/__init__.py rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/data/delete/ghg.py (85%) create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data/process/__init__.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data/process/ghg_maghg/__init__.py rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/data/process/ghg_maghg/ghg_rename_files_and_processing.py (100%) create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data/publish/__init__.py rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/data/publish/publish_3857_13_novemeber.py (99%) rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/data/publish/publish_3857_maghg.py (97%) create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data/update/__init__.py rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/data/update/update_metadata.py (94%) create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data_storage/__init__.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/__init__.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/ghg_maghg/__init__.py rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py (100%) create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/nov_13/__init__.py create mode 100644 geobricks_data_scripts/demo/old_ghg/ghg/data_storage/publish/__init__.py rename geobricks_data_scripts/demo/{ => old_ghg}/ghg/data_storage/publish/publish_4326_maghg.py (94%) create mode 100644 geobricks_data_scripts/demo/old_ghg/utils/.gitignore create mode 100644 geobricks_data_scripts/demo/old_ghg/utils/__init__.py create mode 100644 geobricks_data_scripts/demo/old_ghg/utils/data_manager_util.py diff --git a/geobricks_data_scripts/demo/config/config.py b/geobricks_data_scripts/demo/config/config.py deleted file mode 100644 index 82480f4..0000000 --- a/geobricks_data_scripts/demo/config/config.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging -import os -import json - -settings = { - - "settings": { - - - # To be used by Flask: DEVELOPMENT ONLY - "debug": True, - - # Flask host: DEVELOPMENT ONLY - "host": "localhost", - - # Flask port: DEVELOPMENT ONLY - "port": 5555, - - # Logging configurations - "logging": { - "level": logging.ERROR, - "format": "%(asctime)s | %(levelname)-8s | %(name)-20s | Line: %(lineno)-5d | %(message)s", - "datefmt": "%d-%m-%Y | %H:%M:%s" - }, - - # Email configurations - "email": { - "settings": "path_to_the_email.json", - "user": None, - "password": None - }, - - # Folders - "folders": { - "config": "config/", - "tmp": "tmp_path", - "geoserver_datadir": "/home/vortex/programs/SERVERS/tomcat_geoservers/geoservers-test/data/", - "distribution": "distribution_folder", - "storage": "storage_folder" - }, - - # Geoserver - "geoserver": { - "geoserver_master": "http://lprapp16.fao.org:20100/geoserver/rest", - "geoserver_slaves": [], - "username": "fenix", - "password": "Fenix2014", - }, - - # Metadata - "metadata": { - "url_create_metadata": "http://fenix.fao.org/d3s_dev/msd/resources/metadata", - "url_get_metadata_uid": "http://fenix.fao.org/d3s_dev/msd/resources/metadata/uid/", - - # delete metadata - "url_delete_metadata": "http://fenix.fao.org/d3s_dev/msd/resources/metadata/uid/", - - # get metadata - "url_get_metadata": "http://fenix.fao.org/d3s_dev/msd/resources/find", - "url_get_metadata_full": "http://fenix.fao.org/d3s_dev/msd/resources/metadata/uid/?full=true&dsd=true", - - # coding system - "url_create_coding_system": "http://fenix.fao.org/d3s_dev/msd/resources", - "url_data_coding_system": "http://fenix.fao.org/d3s_dev/msd/resources/data/uid/", - - # DSD - "url_overwrite_dsd_rid": "http://fenix.fao.org/d3s_dev/msd/resources/dsd" - } - } -} - - -# Setting email adderess from configuration file -def set_email_settings(): - if os.path.isfile(settings["settings"]["email"]["settings"]): - settings["settings"]["email"] = json.loads(open(settings["settings"]["email"]["settings"]).read()) -set_email_settings() \ No newline at end of file diff --git a/geobricks_data_scripts/demo/common_data/__init__.py b/geobricks_data_scripts/demo/fenix/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/common_data/__init__.py rename to geobricks_data_scripts/demo/fenix/__init__.py diff --git a/geobricks_data_scripts/demo/common_data/coding_system/__init__.py b/geobricks_data_scripts/demo/fenix/check_consistency/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/common_data/coding_system/__init__.py rename to geobricks_data_scripts/demo/fenix/check_consistency/__init__.py diff --git a/geobricks_data_scripts/demo/fenix/check_consistency/check_consistency.py b/geobricks_data_scripts/demo/fenix/check_consistency/check_consistency.py new file mode 100644 index 0000000..14d369c --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/check_consistency/check_consistency.py @@ -0,0 +1,14 @@ +from geobricks_common.core.log import logger +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager + +log = logger(__file__) + +data_manager = get_data_manager() + +result = data_manager.check_consistency() + +for r in result["storage"]: + log.info(r) + +for r in result["geoserver"]: + log.info(r) \ No newline at end of file diff --git a/geobricks_data_scripts/demo/config/__init__.py b/geobricks_data_scripts/demo/fenix/coding_system/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/config/__init__.py rename to geobricks_data_scripts/demo/fenix/coding_system/__init__.py diff --git a/geobricks_data_scripts/demo/ghg/coding_system/create_cs_csv.py b/geobricks_data_scripts/demo/fenix/coding_system/create_cs_csv.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/coding_system/create_cs_csv.py rename to geobricks_data_scripts/demo/fenix/coding_system/create_cs_csv.py diff --git a/geobricks_data_scripts/demo/ghg/coding_system/csv/cs_ghg.csv b/geobricks_data_scripts/demo/fenix/coding_system/csv/cs_ghg.csv similarity index 100% rename from geobricks_data_scripts/demo/ghg/coding_system/csv/cs_ghg.csv rename to geobricks_data_scripts/demo/fenix/coding_system/csv/cs_ghg.csv diff --git a/geobricks_data_scripts/demo/ghg/coding_system/csv/cs_maghg.csv b/geobricks_data_scripts/demo/fenix/coding_system/csv/cs_maghg.csv similarity index 100% rename from geobricks_data_scripts/demo/ghg/coding_system/csv/cs_maghg.csv rename to geobricks_data_scripts/demo/fenix/coding_system/csv/cs_maghg.csv diff --git a/geobricks_data_scripts/demo/fenix/coding_system/csv/final.csv b/geobricks_data_scripts/demo/fenix/coding_system/csv/final.csv new file mode 100644 index 0000000..90c32c6 --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/coding_system/csv/final.csv @@ -0,0 +1,23 @@ +cultivation_organic_soils_-_croplands,Cultivation Organic Soils - Croplands +burned_areas_-_barren_or_sparsely_vegetated,Burned Areas - Barren or sparsely vegetated +burned_areas_-_closed_shrubland,Burned Areas - Closed shrubland +burned_areas_-_croplands,Burned Areas - Croplands +burned_areas_-_deciduous_broadleaf_forest,Burned Areas - Deciduous Broadleaf forest +burned_areas_-_deciduous_needleleaf_forest,Burned Areas - Deciduous Needleleaf forest +burned_areas_-_evergreen_broadleaf_forest,Burned Areas - Evergreen Broadleaf forest +burned_areas_-_evergreen_needleleaf_forest,Burned Areas - Evergreen Needleleaf forest +burned_areas_-_grassland,Burned Areas - Grassland +burned_areas_-_humid_tropical_forest,Burned Areas - Humid Tropical Forest +burned_areas_-_mixed_forest,Burned Areas - Mixed forest +burned_areas_-_open_shrubland,Burned Areas - Open shrubland +burned_areas_-_organic_soils,Burned Areas - Organic soils +burned_areas_-_other_forest,Burned Areas - Other forest +burned_areas_-_savanna,Burned Areas - Savanna +burned_areas_-_unclassified,Burned Areas - Unclassified +burned_areas_-_urban_and_built-up,Burned Areas - Urban and built-up +burned_areas_-_woody_savanna,Burned Areas - Woody savanna +global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 +global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) +gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 +jrc_climate_zone,JRC climate zone +modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD \ No newline at end of file diff --git a/geobricks_data_scripts/demo/fenix/coding_system/csv/final_without_maghg_and_soil.csv b/geobricks_data_scripts/demo/fenix/coding_system/csv/final_without_maghg_and_soil.csv new file mode 100644 index 0000000..403530d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/coding_system/csv/final_without_maghg_and_soil.csv @@ -0,0 +1,23 @@ +cultivation_organic_soils_-_croplands,Cultivation Organic Soils - Croplands +burned_areas_-_barren_or_sparsely_vegetated,Burned Areas - Barren or sparsely vegetated +burned_areas_-_closed_shrubland,Burned Areas - Closed shrubland +burned_areas_-_croplands,Burned Areas - Croplands +burned_areas_-_deciduous_broadleaf_forest,Burned Areas - Deciduous Broadleaf forest +burned_areas_-_deciduous_needleleaf_forest,Burned Areas - Deciduous Needleleaf forest +burned_areas_-_evergreen_broadleaf_forest,Burned Areas - Evergreen Broadleaf forest +burned_areas_-_evergreen_needleleaf_forest,Burned Areas - Evergreen Needleleaf forest +burned_areas_-_grassland,Burned Areas - Grassland +burned_areas_-_humid_tropical_forest,Burned Areas - Humid Tropical Forest +burned_areas_-_mixed_forest,Burned Areas - Mixed forest +burned_areas_-_open_shrubland,Burned Areas - Open shrubland +burned_areas_-_organic_soils,Burned Areas - Organic soils +burned_areas_-_other_forest,Burned Areas - Other forest +burned_areas_-_savanna,Burned Areas - Savanna +burned_areas_-_unclassified,Burned Areas - Unclassified +burned_areas_-_urban_and_built-up,Burned Areas - Urban and built-up +burned_areas_-_woody_savanna,Burned Areas - Woody savanna +global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 +global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) +gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 +jrc_climate_zone,JRC climate zone +modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD diff --git a/geobricks_data_scripts/demo/ghg/coding_system/publish_final_cs.py b/geobricks_data_scripts/demo/fenix/coding_system/publish_final_cs.py similarity index 91% rename from geobricks_data_scripts/demo/ghg/coding_system/publish_final_cs.py rename to geobricks_data_scripts/demo/fenix/coding_system/publish_final_cs.py index 0d975e3..322cc4f 100644 --- a/geobricks_data_scripts/demo/ghg/coding_system/publish_final_cs.py +++ b/geobricks_data_scripts/demo/fenix/coding_system/publish_final_cs.py @@ -2,7 +2,8 @@ import json from geobricks_metadata_manager.core.metadata_manager_d3s_core import MetadataManager -from geobricks_data_scripts.demo.config.config import settings +from geobricks_data_scripts.demo.fenix.config.config import settings + metadata_manager = MetadataManager(settings) diff --git a/geobricks_data_scripts/demo/ghg/coding_system/publish_maghg_cs.py b/geobricks_data_scripts/demo/fenix/coding_system/publish_maghg_cs.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/coding_system/publish_maghg_cs.py rename to geobricks_data_scripts/demo/fenix/coding_system/publish_maghg_cs.py diff --git a/geobricks_data_scripts/demo/ghg/__init__.py b/geobricks_data_scripts/demo/fenix/common_data/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/__init__.py rename to geobricks_data_scripts/demo/fenix/common_data/__init__.py diff --git a/geobricks_data_scripts/demo/ghg/coding_system/__init__.py b/geobricks_data_scripts/demo/fenix/common_data/coding_system/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/coding_system/__init__.py rename to geobricks_data_scripts/demo/fenix/common_data/coding_system/__init__.py diff --git a/geobricks_data_scripts/demo/common_data/coding_system/update_mapProjections.py b/geobricks_data_scripts/demo/fenix/common_data/coding_system/update_mapProjections.py similarity index 88% rename from geobricks_data_scripts/demo/common_data/coding_system/update_mapProjections.py rename to geobricks_data_scripts/demo/fenix/common_data/coding_system/update_mapProjections.py index c92b6cf..cbb87cf 100644 --- a/geobricks_data_scripts/demo/common_data/coding_system/update_mapProjections.py +++ b/geobricks_data_scripts/demo/fenix/common_data/coding_system/update_mapProjections.py @@ -1,4 +1,4 @@ -from geobricks_data_scripts.demo.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager data_manager = get_data_manager() diff --git a/geobricks_data_scripts/demo/fenix/config/.gitignore b/geobricks_data_scripts/demo/fenix/config/.gitignore new file mode 100644 index 0000000..56c1683 --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/config/.gitignore @@ -0,0 +1,2 @@ +*.pyc +*.py diff --git a/geobricks_data_scripts/demo/ghg/data/__init__.py b/geobricks_data_scripts/demo/fenix/data/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data/__init__.py rename to geobricks_data_scripts/demo/fenix/data/__init__.py diff --git a/geobricks_data_scripts/demo/ghg/data/delete/__init__.py b/geobricks_data_scripts/demo/fenix/data/earthstat/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data/delete/__init__.py rename to geobricks_data_scripts/demo/fenix/data/earthstat/__init__.py diff --git a/geobricks_data_scripts/demo/ghg/data/process/__init__.py b/geobricks_data_scripts/demo/fenix/data/earthstat/delete/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data/process/__init__.py rename to geobricks_data_scripts/demo/fenix/data/earthstat/delete/__init__.py diff --git a/geobricks_data_scripts/demo/fenix/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/demo/fenix/data/earthstat/delete/delete_all_earthstat.py new file mode 100644 index 0000000..6902f79 --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/earthstat/delete/delete_all_earthstat.py @@ -0,0 +1,11 @@ +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +for layer in layers: + try: + if "earthstat" in layer["dsd"]["workspace"]: + data_manager.delete(layer["uid"]) + except Exception, e: + print "Error", e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/demo/ghg/data/process/ghg_maghg/__init__.py b/geobricks_data_scripts/demo/fenix/data/earthstat/publish/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data/process/ghg_maghg/__init__.py rename to geobricks_data_scripts/demo/fenix/data/earthstat/publish/__init__.py diff --git a/geobricks_data_scripts/demo/fenix/data/earthstat/publish/publish_harvest.py b/geobricks_data_scripts/demo/fenix/data/earthstat/publish/publish_harvest.py new file mode 100644 index 0000000..5b8d8fb --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/earthstat/publish/publish_harvest.py @@ -0,0 +1,15 @@ +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder + + +data_manager = get_data_manager() + +lang = "EN" +workspace = "earthstat" + + +metadatas = harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area/", workspace) +print metadatas +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/", workspace) + + diff --git a/geobricks_data_scripts/demo/ghg/data/publish/__init__.py b/geobricks_data_scripts/demo/fenix/data/ghg/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data/publish/__init__.py rename to geobricks_data_scripts/demo/fenix/data/ghg/__init__.py diff --git a/geobricks_data_scripts/demo/ghg/data/update/__init__.py b/geobricks_data_scripts/demo/fenix/data/ghg/delete/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data/update/__init__.py rename to geobricks_data_scripts/demo/fenix/data/ghg/delete/__init__.py diff --git a/geobricks_data_scripts/demo/fenix/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/demo/fenix/data/ghg/delete/data_manager_delete_all_example.py new file mode 100644 index 0000000..6092615 --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/ghg/delete/data_manager_delete_all_example.py @@ -0,0 +1,11 @@ +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +for layer in layers: + try: + print layer["uid"] + data_manager.delete(layer["uid"]) + except Exception, e: + print e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/demo/ghg/data_storage/__init__.py b/geobricks_data_scripts/demo/fenix/data/ghg/publish/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data_storage/__init__.py rename to geobricks_data_scripts/demo/fenix/data/ghg/publish/__init__.py diff --git a/geobricks_data_scripts/demo/fenix/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/demo/fenix/data/ghg/publish/publish_GHG_data.py new file mode 100644 index 0000000..4075bef --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/ghg/publish/publish_GHG_data.py @@ -0,0 +1,63 @@ +import os +import glob + +from geobricks_common.core.log import logger +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder + + +log = logger(__file__) + +data_manager = get_data_manager() + +workspace = "ghg" + + +def default_import(src_folder): + log.info("GLC2000: " + src_folder) + metadatas = harvest_folder(data_manager, src_folder, workspace, True, True, True) + + +def burned_areas(src_folder): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + folder = os.path.join(folder, "output") + log.info(folder) + metadatas = harvest_folder(data_manager, folder, workspace, True, True, True) + log.info(metadatas) + for metadata in metadatas: + log.info(metadata) + if "3857" in metadata["dsd"]["layerName"]: + log.info("here") + # log.info"metadata["dsd"]["datasource"]) + data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], workspace, "burned_areas_EN") + + +# used just to set the style to modis that was wrong +def set_style_modis(): + data_manager.geoserver_manager.set_style("modis_land_cover_mod12q1_umd_500m_2009_3857", workspace, "modis_-_land_cover_type_umd_EN") + + +if __name__ == '__main__': + base_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/TO_PUBLISH" + + #Cultivation organic soils + default_import(os.path.join(base_folder, "Cultivation_organic_soils_croplands", "Cultivation_organic_soils_-_croplands")) + + # Burned areas + burned_areas(os.path.join(base_folder, "GFED4_BURNEDAREAS_BY_LANDCOVER")) + + # GLC 2000 (TODO: set the style) + #default_import(os.path.join(base_folder, "Global_Land_Cover_2000_(GLC_2000)", "Global_Land_Cover_2000_(GLC_2000)")) + + # griddedlivestock + default_import(os.path.join(base_folder, "GriddedLivestock", "gridded_livestock_of_the_world_v_201")) + + # JRC Climate zone + default_import(os.path.join(base_folder, "JRC_CLIMATE_ZONE", "JRC_climate_zone")) + + # MODIS_-_Land_Cover_type_UMD + default_import(os.path.join(base_folder, "MODIS_land_cover", "MODIS_-_Land_Cover_type_UMD")) + + + diff --git a/geobricks_data_scripts/demo/config/.gitignore b/geobricks_data_scripts/demo/fenix/utils/.gitignore similarity index 100% rename from geobricks_data_scripts/demo/config/.gitignore rename to geobricks_data_scripts/demo/fenix/utils/.gitignore diff --git a/geobricks_data_scripts/demo/ghg/data_storage/process/__init__.py b/geobricks_data_scripts/demo/fenix/utils/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data_storage/process/__init__.py rename to geobricks_data_scripts/demo/fenix/utils/__init__.py diff --git a/geobricks_data_scripts/demo/utils/data_manager_util.py b/geobricks_data_scripts/demo/fenix/utils/data_manager_util.py similarity index 71% rename from geobricks_data_scripts/demo/utils/data_manager_util.py rename to geobricks_data_scripts/demo/fenix/utils/data_manager_util.py index 6b7b54e..6395fea 100644 --- a/geobricks_data_scripts/demo/utils/data_manager_util.py +++ b/geobricks_data_scripts/demo/fenix/utils/data_manager_util.py @@ -1,5 +1,5 @@ from geobricks_data_manager.core.data_manager_core import DataManager -from geobricks_data_scripts.demo.config.config import settings +from geobricks_data_scripts.demo.fenix.config.config import settings # get the default data manager for demo db def get_data_manager(): diff --git a/geobricks_data_scripts/demo/utils/.gitignore b/geobricks_data_scripts/demo/old_ghg/.gitignore similarity index 100% rename from geobricks_data_scripts/demo/utils/.gitignore rename to geobricks_data_scripts/demo/old_ghg/.gitignore diff --git a/geobricks_data_scripts/demo/ghg/data_storage/process/ghg_maghg/__init__.py b/geobricks_data_scripts/demo/old_ghg/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data_storage/process/ghg_maghg/__init__.py rename to geobricks_data_scripts/demo/old_ghg/__init__.py diff --git a/geobricks_data_scripts/demo/ghg/data_storage/process/nov_13/__init__.py b/geobricks_data_scripts/demo/old_ghg/common_data/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data_storage/process/nov_13/__init__.py rename to geobricks_data_scripts/demo/old_ghg/common_data/__init__.py diff --git a/geobricks_data_scripts/demo/ghg/data_storage/publish/__init__.py b/geobricks_data_scripts/demo/old_ghg/common_data/coding_system/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data_storage/publish/__init__.py rename to geobricks_data_scripts/demo/old_ghg/common_data/coding_system/__init__.py diff --git a/geobricks_data_scripts/demo/old_ghg/common_data/coding_system/update_mapProjections.py b/geobricks_data_scripts/demo/old_ghg/common_data/coding_system/update_mapProjections.py new file mode 100644 index 0000000..3229a06 --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/common_data/coding_system/update_mapProjections.py @@ -0,0 +1,33 @@ +from geobricks_data_scripts.demo.old_ghg.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() + +cs = { + "metadata": { + "uid": "mapProjections", + "version": "1.0" + }, + "data": [ + { + "code": "EPSG:4326", + "title": { + "EN": "Latlong" + } + }, + { + "code": "EPSG:3857", + "title": { + "EN": "Spherical Mercator projection" + } + }, + { + "code": "EPSG:900913", + "title": { + "EN": "Equivalent to Spherical Mercator Projection" + } + } + ] +} + +# Update coding system +data_manager.metadata_manager.update_coding_system(cs, True) \ No newline at end of file diff --git a/geobricks_data_scripts/demo/old_ghg/config/.gitignore b/geobricks_data_scripts/demo/old_ghg/config/.gitignore new file mode 100644 index 0000000..56c1683 --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/config/.gitignore @@ -0,0 +1,2 @@ +*.pyc +*.py diff --git a/geobricks_data_scripts/demo/utils/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/utils/__init__.py rename to geobricks_data_scripts/demo/old_ghg/ghg/__init__.py diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/create_cs_csv.py b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/create_cs_csv.py new file mode 100644 index 0000000..3ff0dde --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/create_cs_csv.py @@ -0,0 +1,48 @@ +import csv + +items = [ + "Cultivation Organic Soils", + "GFED4 Burned Areas", + "GFED4 Burned Areas - Barren or sparsely vegetated", + "GFED4 Burned Areas - Closed shrubland", + "GFED4 Burned Areas - Croplands", + "GFED4 Burned Areas - Deciduous Broadleaf forest", + "GFED4 Burned Areas - Deciduous Needleleaf forest", + "GFED4 Burned Areas - Evergreen Broadleaf forest", + "GFED4 Burned Areas - Evergreen Needleleaf forest", + "GFED4 Burned Areas - Grassland", + "GFED4 Burned Areas - Humid Tropical Forest", + "GFED4 Burned Areas - Mixed forest", + "GFED4 Burned Areas - Open shrubland", + "GFED4 Burned Areas - Organic soils", + "GFED4 Burned Areas - Other forest", + "GFED4 Burned Areas - Savanna", + "GFED4 Burned Areas - Unclassified", + "GFED4 Burned Areas - Urban and built-up", + "GFED4 Burned Areas - Woody savanna", + "Global Ecological Zones (GEZ) 2010", + "Global Land Cover 2000 (GLC2000)", + "Gridded Livestock of the World v. 2.01", + "Harmonized World Soil Database - Organic soils", + "JRC climate zone", + "JRC climate zone (4326)", + "MODIS - Land Cover Type UMD" +] + + +final = [] +with open('csv/cs_ghg.csv', 'wb') as csvfile: + w = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + for i in items: + s = i.lower().replace(" ", "_") + w.writerow([s, i]) + final.append([s, i]) +print final + + + + + + + + diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/cs_ghg.csv b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/cs_ghg.csv new file mode 100644 index 0000000..708565a --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/cs_ghg.csv @@ -0,0 +1,25 @@ +cultivation_organic_soils,Cultivation Organic Soils +gfed4_burned_areas,GFED4 Burned Areas +gfed4_burned_areas_-_barren_or_sparsely_vegetated,GFED4 Burned Areas - Barren or sparsely vegetated +gfed4_burned_areas_-_closed_shrubland,GFED4 Burned Areas - Closed shrubland +gfed4_burned_areas_-_croplands,GFED4 Burned Areas - Croplands +gfed4_burned_areas_-_deciduous_broadleaf_forest,GFED4 Burned Areas - Deciduous Broadleaf forest +gfed4_burned_areas_-_deciduous_needleleaf_forest,GFED4 Burned Areas - Deciduous Needleleaf forest +gfed4_burned_areas_-_evergreen_broadleaf_forest,GFED4 Burned Areas - Evergreen Broadleaf forest +gfed4_burned_areas_-_evergreen_needleleaf_forest,GFED4 Burned Areas - Evergreen Needleleaf forest +gfed4_burned_areas_-_grassland,GFED4 Burned Areas - Grassland +gfed4_burned_areas_-_humid_tropical_forest,GFED4 Burned Areas - Humid Tropical Forest +gfed4_burned_areas_-_mixed_forest,GFED4 Burned Areas - Mixed forest +gfed4_burned_areas_-_open_shrubland,GFED4 Burned Areas - Open shrubland +gfed4_burned_areas_-_organic_soils,GFED4 Burned Areas - Organic soils +gfed4_burned_areas_-_other_forest,GFED4 Burned Areas - Other forest +gfed4_burned_areas_-_savanna,GFED4 Burned Areas - Savanna +gfed4_burned_areas_-_unclassified,GFED4 Burned Areas - Unclassified +gfed4_burned_areas_-_urban_and_built-up,GFED4 Burned Areas - Urban and built-up +gfed4_burned_areas_-_woody_savanna,GFED4 Burned Areas - Woody savanna +global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 +global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) +gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 +harmonized_world_soil_database_-_organic_soils,Harmonized World Soil Database - Organic soils +jrc_climate_zone,JRC climate zone +modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/cs_maghg.csv b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/cs_maghg.csv new file mode 100644 index 0000000..959a829 --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/cs_maghg.csv @@ -0,0 +1,45 @@ +dm_burning_grasslands,DM Burning Grasslands +ch4_emissions_burning_woodysavanna,CH4 Emissions Burning WoodySavanna +n2o_emissions_burning_grasslands,N2O Emissions Burning Grasslands +n2o_emissions_burning_woodysavanna,N2O Emissions Burning WoodySavanna +co2_emissions_burning_closedshrublands,CO2 Emissions Burning ClosedShrublands +co2_emissions_burning_openshrublands,CO2 Emissions Burning OpenShrublands +co2_emissions_burning_humidtropicalforests,CO2 Emissions Burning HumidTropicalForests +dm_burning_openshrublands,DM Burning OpenShrublands +n2o_emissions_burning_otherforests,N2O Emissions Burning OtherForests +n2o_emissions_burning_humidtropicalforests,N2O Emissions Burning HumidTropicalForests +n2o_emissions_burning_savanna,N2O Emissions Burning Savanna +ch4_emissions_burning_humidtropicalforests,CH4 Emissions Burning HumidTropicalForests +ch4_emissions_burning_peatlands,CH4 Emissions Burning Peatlands +n2o_emissions_burning_peatlands,N2O Emissions Burning Peatlands +ch4_emissions_burning_closedshrublands,CH4 Emissions Burning ClosedShrublands +co2_emissions_burning_grasslands,CO2 Emissions Burning Grasslands +ch4_emissions_burning_grasslands,CH4 Emissions Burning Grasslands +openshrublands_fuelbiomass,OpenShrublands FuelBiomass +co2_emissions_burning_woodysavanna,CO2 Emissions Burning WoodySavanna +ch4_emissions_burning_otherforests,CH4 Emissions Burning OtherForests +dm_burning_closedshrublands,DM Burning ClosedShrublands +dm_burning_woodysavanna,DM Burning WoodySavanna +ch4_emissions_burning_savanna,CH4 Emissions Burning Savanna +co2_emissions_burning_otherforests,CO2 Emissions Burning OtherForests +n2o_emissions_burning_closedshrublands,N2O Emissions Burning ClosedShrublands +co2_emissions_burning_peatlands,CO2 Emissions Burning Peatlands +co2_emissions_burning_savanna,CO2 Emissions Burning Savanna +dm_burning_peatlands,DM Burning Peatlands +dm_burning_humidtropicalforests,DM Burning HumidTropicalForests +savanna_fuelbiomass,Savanna FuelBiomass +n2o_emissions_burning_openshrublands,N2O Emissions Burning OpenShrublands +tropical_forests_ch4_ef,Tropical Forests CH4 EF +dm_burning_savanna,DM Burning Savanna +ch4_emissions_burning_openshrublands,CH4 Emissions Burning OpenShrublands +dm_burning_otherforests,DM Burning OtherForests +tropical_forests_fuelbiomass,Tropical Forests FuelBiomass +woodysavanna_fuelbiomass,WoodySavanna FuelBiomass +grasslands_fuelbiomass,Grasslands FuelBiomass +nontropical_forests_fuelbiomass,NonTropical Forests FuelBiomass +tropical_forests_co2_ef,Tropical Forests CO2 EF +extratropical_forests_n2o_ef,ExtraTropical Forests N2O EF +closedshrublands_fuelbiomass,ClosedShrublands FuelBiomass +tropical_forests_n2o_ef,Tropical Forests N2O EF +extratropical_forests_co2_ef,ExtraTropical Forests CO2 EF +extratropical_forests_ch4_ef,ExtraTropical Forests CH4 EF diff --git a/geobricks_data_scripts/demo/ghg/coding_system/csv/final.csv b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/final.csv similarity index 100% rename from geobricks_data_scripts/demo/ghg/coding_system/csv/final.csv rename to geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/final.csv diff --git a/geobricks_data_scripts/demo/ghg/coding_system/csv/final_without_maghg_and_soil.csv b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/final_without_maghg_and_soil.csv similarity index 100% rename from geobricks_data_scripts/demo/ghg/coding_system/csv/final_without_maghg_and_soil.csv rename to geobricks_data_scripts/demo/old_ghg/ghg/coding_system/csv/final_without_maghg_and_soil.csv diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/publish_final_cs.py b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/publish_final_cs.py new file mode 100644 index 0000000..d30f0fd --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/publish_final_cs.py @@ -0,0 +1,44 @@ +import csv +import json + +from geobricks_metadata_manager.core.metadata_manager_d3s_core import MetadataManager +from geobricks_data_scripts.demo.old_ghg.config.config import settings + + +metadata_manager = MetadataManager(settings) + +print metadata_manager + +cs = { + "metadata": { + "uid": "layer_products", + "version": "1.0", + "title": { + "EN": "Layers" + }, + "meContent": { + "resourceRepresentationType": "codelist" + } + }, + "data": [ + + ] +} + + +with open('csv/final.csv', 'rb') as csvfile: + rows = csv.reader(csvfile) + for row in rows: + cs["data"].append({ + "code": row[0], + "title": { + "EN": row[1] + } + }) + + +print json.dumps(cs) +print metadata_manager.publish_coding_system(cs, True) + + + diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/publish_maghg_cs.py b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/publish_maghg_cs.py new file mode 100644 index 0000000..f4a93fb --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/coding_system/publish_maghg_cs.py @@ -0,0 +1,26 @@ +import glob +import os +import csv + +files = glob.glob("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/*.tif") + +cached_codes = [] +codes = [] +for f in files: + if "_3857" in f: + if "20" in f or "19" in f: + # it contains a date + folder, filename = os.path.split(f.rsplit("_", 2)[0]) + else: + folder, filename = os.path.split(f.rsplit("_", 1)[0]) + if filename not in cached_codes: + cached_codes.append(filename) + codes.append([filename.lower().replace(" ","_"), filename.replace("_", " ")]) + + +with open('csv/cs_maghg.csv', 'wb') as csvfile: + w = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + for code in codes: + w.writerow(code) + + diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data/delete/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/ghg/data/delete/ghg.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/delete/ghg.py similarity index 85% rename from geobricks_data_scripts/demo/ghg/data/delete/ghg.py rename to geobricks_data_scripts/demo/old_ghg/ghg/data/delete/ghg.py index d827155..797a88f 100644 --- a/geobricks_data_scripts/demo/ghg/data/delete/ghg.py +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/delete/ghg.py @@ -1,7 +1,4 @@ -from geobricks_data_scripts.demo.utils.data_manager_util import get_data_manager -from geobricks_data_manager.core.data_manager_core import DataManager -import csv - +from geobricks_data_scripts.demo.old_ghg.utils.data_manager_util import get_data_manager data_manager = get_data_manager() diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data/process/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/process/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/process/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data/process/ghg_maghg/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/process/ghg_maghg/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/process/ghg_maghg/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/ghg/data/process/ghg_maghg/ghg_rename_files_and_processing.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/process/ghg_maghg/ghg_rename_files_and_processing.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data/process/ghg_maghg/ghg_rename_files_and_processing.py rename to geobricks_data_scripts/demo/old_ghg/ghg/data/process/ghg_maghg/ghg_rename_files_and_processing.py diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data/publish/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/ghg/data/publish/publish_3857_13_novemeber.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/publish/publish_3857_13_novemeber.py similarity index 99% rename from geobricks_data_scripts/demo/ghg/data/publish/publish_3857_13_novemeber.py rename to geobricks_data_scripts/demo/old_ghg/ghg/data/publish/publish_3857_13_novemeber.py index 1853068..54e8db4 100644 --- a/geobricks_data_scripts/demo/ghg/data/publish/publish_3857_13_novemeber.py +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/publish/publish_3857_13_novemeber.py @@ -6,7 +6,8 @@ from geobricks_data_scripts.utils.filesystem import get_filename from geobricks_common.core.date import get_daterange -from geobricks_data_scripts.demo.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.demo.old_ghg.utils.data_manager_util import get_data_manager + # data manger data_manager = get_data_manager() diff --git a/geobricks_data_scripts/demo/ghg/data/publish/publish_3857_maghg.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/publish/publish_3857_maghg.py similarity index 97% rename from geobricks_data_scripts/demo/ghg/data/publish/publish_3857_maghg.py rename to geobricks_data_scripts/demo/old_ghg/ghg/data/publish/publish_3857_maghg.py index fdd151d..28af312 100644 --- a/geobricks_data_scripts/demo/ghg/data/publish/publish_3857_maghg.py +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/publish/publish_3857_maghg.py @@ -2,12 +2,11 @@ import glob import calendar import datetime -import json import os -from geobricks_data_scripts.utils.filesystem import get_filename from geobricks_common.core.date import get_daterange -from geobricks_data_scripts.demo.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.demo.old_ghg.utils.data_manager_util import get_data_manager + # data manager data_manager = get_data_manager() diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data/update/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/update/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/update/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/ghg/data/update/update_metadata.py b/geobricks_data_scripts/demo/old_ghg/ghg/data/update/update_metadata.py similarity index 94% rename from geobricks_data_scripts/demo/ghg/data/update/update_metadata.py rename to geobricks_data_scripts/demo/old_ghg/ghg/data/update/update_metadata.py index 2cf1770..0dd018a 100644 --- a/geobricks_data_scripts/demo/ghg/data/update/update_metadata.py +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data/update/update_metadata.py @@ -1,4 +1,4 @@ -from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.demo.old_ghg.utils.data_manager_util import get_data_manager data_manager = get_data_manager() diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/ghg_maghg/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/ghg_maghg/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/ghg_maghg/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py similarity index 100% rename from geobricks_data_scripts/demo/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py rename to geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/ghg_maghg/maghg_create_folders_move_files_rename_it.py diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/nov_13/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/nov_13/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/process/nov_13/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/publish/__init__.py b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/ghg/data_storage/publish/publish_4326_maghg.py b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/publish/publish_4326_maghg.py similarity index 94% rename from geobricks_data_scripts/demo/ghg/data_storage/publish/publish_4326_maghg.py rename to geobricks_data_scripts/demo/old_ghg/ghg/data_storage/publish/publish_4326_maghg.py index fcc194a..12c6916 100644 --- a/geobricks_data_scripts/demo/ghg/data_storage/publish/publish_4326_maghg.py +++ b/geobricks_data_scripts/demo/old_ghg/ghg/data_storage/publish/publish_4326_maghg.py @@ -1,7 +1,9 @@ import os import glob + from geobricks_common.core.filesystem import sanitize_name, get_filename -from geobricks_data_scripts.demo.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.demo.old_ghg.utils.data_manager_util import get_data_manager + data_manager = get_data_manager() diff --git a/geobricks_data_scripts/demo/old_ghg/utils/.gitignore b/geobricks_data_scripts/demo/old_ghg/utils/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/utils/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/geobricks_data_scripts/demo/old_ghg/utils/__init__.py b/geobricks_data_scripts/demo/old_ghg/utils/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/utils/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/old_ghg/utils/data_manager_util.py b/geobricks_data_scripts/demo/old_ghg/utils/data_manager_util.py new file mode 100644 index 0000000..ffb408c --- /dev/null +++ b/geobricks_data_scripts/demo/old_ghg/utils/data_manager_util.py @@ -0,0 +1,6 @@ +from geobricks_data_manager.core.data_manager_core import DataManager +from geobricks_data_scripts.demo.old_ghg.config.config import settings + + # get the default data manager for demo db +def get_data_manager(): + return DataManager(settings) \ No newline at end of file From 06dc09f21e8728544755b9859fd076acce3d3383 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Fri, 13 Feb 2015 11:16:19 +0100 Subject: [PATCH 15/39] removed test structure --- geobricks_data_scripts/README.md | 0 .../check_consistency}/__init__.py | 0 .../check_consistency/check_consistency.py | 11 +++++ .../{test => prod}/config/.gitignore | 0 .../common_data => prod/config}/__init__.py | 0 .../{test => prod}/config/config.py | 22 ++++----- .../coding_system => prod/data}/__init__.py | 0 .../data/earthstat}/__init__.py | 0 .../data/earthstat/delete}/__init__.py | 0 .../earthstat/delete/delete_all_earthstat.py | 0 .../data/earthstat/publish}/__init__.py | 0 .../data/earthstat/publish/publish_harvest.py | 0 .../delete => prod/data/ghg}/__init__.py | 0 .../data/ghg/delete}/__init__.py | 0 .../delete/data_manager_delete_all_example.py | 0 .../ghg => prod/data/ghg/publish}/__init__.py | 0 .../data/ghg/publish/publish_GHG_data.py | 0 .../{test => prod}/utils/.gitignore | 0 .../ghg/delete => prod/utils}/__init__.py | 0 .../{test => prod}/utils/data_manager_util.py | 0 geobricks_data_scripts/test/__init__.py | 4 -- .../test/coding_system/create_cs_csv.py | 48 ------------------- .../test/coding_system/csv/cs_ghg.csv | 25 ---------- .../test/coding_system/csv/cs_maghg.csv | 45 ----------------- .../test/coding_system/csv/final.csv | 23 --------- .../csv/final_without_maghg_and_soil.csv | 23 --------- .../test/coding_system/publish_final_cs.py | 43 ----------------- .../test/coding_system/publish_maghg_cs.py | 26 ---------- .../coding_system/update_mapProjections.py | 33 ------------- .../test/data/ghg/publish/__init__.py | 1 - geobricks_data_scripts/test/utils/__init__.py | 1 - 31 files changed, 22 insertions(+), 283 deletions(-) create mode 100644 geobricks_data_scripts/README.md rename geobricks_data_scripts/{test/coding_system => prod/check_consistency}/__init__.py (100%) create mode 100644 geobricks_data_scripts/prod/check_consistency/check_consistency.py rename geobricks_data_scripts/{test => prod}/config/.gitignore (100%) rename geobricks_data_scripts/{test/common_data => prod/config}/__init__.py (100%) rename geobricks_data_scripts/{test => prod}/config/config.py (72%) rename geobricks_data_scripts/{test/common_data/coding_system => prod/data}/__init__.py (100%) rename geobricks_data_scripts/{test/config => prod/data/earthstat}/__init__.py (100%) rename geobricks_data_scripts/{test/data => prod/data/earthstat/delete}/__init__.py (100%) rename geobricks_data_scripts/{test => prod}/data/earthstat/delete/delete_all_earthstat.py (100%) rename geobricks_data_scripts/{test/data/earthstat => prod/data/earthstat/publish}/__init__.py (100%) rename geobricks_data_scripts/{test => prod}/data/earthstat/publish/publish_harvest.py (100%) rename geobricks_data_scripts/{test/data/earthstat/delete => prod/data/ghg}/__init__.py (100%) rename geobricks_data_scripts/{test/data/earthstat/publish => prod/data/ghg/delete}/__init__.py (100%) rename geobricks_data_scripts/{test => prod}/data/ghg/delete/data_manager_delete_all_example.py (100%) rename geobricks_data_scripts/{test/data/ghg => prod/data/ghg/publish}/__init__.py (100%) rename geobricks_data_scripts/{test => prod}/data/ghg/publish/publish_GHG_data.py (100%) rename geobricks_data_scripts/{test => prod}/utils/.gitignore (100%) rename geobricks_data_scripts/{test/data/ghg/delete => prod/utils}/__init__.py (100%) rename geobricks_data_scripts/{test => prod}/utils/data_manager_util.py (100%) delete mode 100644 geobricks_data_scripts/test/__init__.py delete mode 100644 geobricks_data_scripts/test/coding_system/create_cs_csv.py delete mode 100644 geobricks_data_scripts/test/coding_system/csv/cs_ghg.csv delete mode 100644 geobricks_data_scripts/test/coding_system/csv/cs_maghg.csv delete mode 100644 geobricks_data_scripts/test/coding_system/csv/final.csv delete mode 100644 geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv delete mode 100644 geobricks_data_scripts/test/coding_system/publish_final_cs.py delete mode 100644 geobricks_data_scripts/test/coding_system/publish_maghg_cs.py delete mode 100644 geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py delete mode 100644 geobricks_data_scripts/test/data/ghg/publish/__init__.py delete mode 100644 geobricks_data_scripts/test/utils/__init__.py diff --git a/geobricks_data_scripts/README.md b/geobricks_data_scripts/README.md new file mode 100644 index 0000000..e69de29 diff --git a/geobricks_data_scripts/test/coding_system/__init__.py b/geobricks_data_scripts/prod/check_consistency/__init__.py similarity index 100% rename from geobricks_data_scripts/test/coding_system/__init__.py rename to geobricks_data_scripts/prod/check_consistency/__init__.py diff --git a/geobricks_data_scripts/prod/check_consistency/check_consistency.py b/geobricks_data_scripts/prod/check_consistency/check_consistency.py new file mode 100644 index 0000000..a3a5c44 --- /dev/null +++ b/geobricks_data_scripts/prod/check_consistency/check_consistency.py @@ -0,0 +1,11 @@ +import os +import glob +from geobricks_common.core.log import logger +from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder + +log = logger(__file__) + +data_manager = get_data_manager() + +data_manager.check_consistency() diff --git a/geobricks_data_scripts/test/config/.gitignore b/geobricks_data_scripts/prod/config/.gitignore similarity index 100% rename from geobricks_data_scripts/test/config/.gitignore rename to geobricks_data_scripts/prod/config/.gitignore diff --git a/geobricks_data_scripts/test/common_data/__init__.py b/geobricks_data_scripts/prod/config/__init__.py similarity index 100% rename from geobricks_data_scripts/test/common_data/__init__.py rename to geobricks_data_scripts/prod/config/__init__.py diff --git a/geobricks_data_scripts/test/config/config.py b/geobricks_data_scripts/prod/config/config.py similarity index 72% rename from geobricks_data_scripts/test/config/config.py rename to geobricks_data_scripts/prod/config/config.py index 109508d..0e62287 100644 --- a/geobricks_data_scripts/test/config/config.py +++ b/geobricks_data_scripts/prod/config/config.py @@ -41,30 +41,30 @@ # Geoserver "geoserver": { - "geoserver_master": "http://http://fenix.fao.org:20200/geoserver/rest", + "geoserver_master": "http://fenix.fao.org:20200/geoserver/rest", "geoserver_slaves": [], - "username": "admin", - "password": "geoserver", + "username": "fenix", + "password": "Fenix2014", }, # Metadata "metadata": { - "url_create_metadata": "http://localhost:7788/v2/msd/resources/metadata", - "url_get_metadata_uid": "http://localhost:7788/v2/msd/resources/metadata/uid/", + "url_create_metadata": "http://fenix.fao.org/d3s/msd/resources/metadata", + "url_get_metadata_uid": "http://fenix.fao.org/d3s/msd/resources/metadata/uid/", # delete metadata - "url_delete_metadata": "http://localhost:7788/v2/msd/resources/metadata/uid/", + "url_delete_metadata": "http://fenix.fao.org/d3s/msd/resources/metadata/uid/", # get metadata - "url_get_metadata": "http://localhost:7788/v2/msd/resources/find", - "url_get_metadata_full": "http://localhost:7788/v2/msd/resources/metadata/uid/?full=true&dsd=true", + "url_get_metadata": "http://fenix.fao.org/d3s/msd/resources/find", + "url_get_metadata_full": "http://fenix.fao.org/d3s/msd/resources/metadata/uid/?full=true&dsd=true", # coding system - "url_create_coding_system": "http://localhost:7788/v2/msd/resources", - "url_data_coding_system": "http://localhost:7788/v2/msd/resources/data/uid/", + "url_create_coding_system": "http://fenix.fao.org/d3s/msd/resources", + "url_data_coding_system": "http://fenix.fao.org/d3s/msd/resources/data/uid/", # DSD - "url_overwrite_dsd_rid": "http://localhost:7788/v2/msd/resources/dsd" + "url_overwrite_dsd_rid": "http://fenix.fao.org/d3s/msd/resources/dsd" } } } diff --git a/geobricks_data_scripts/test/common_data/coding_system/__init__.py b/geobricks_data_scripts/prod/data/__init__.py similarity index 100% rename from geobricks_data_scripts/test/common_data/coding_system/__init__.py rename to geobricks_data_scripts/prod/data/__init__.py diff --git a/geobricks_data_scripts/test/config/__init__.py b/geobricks_data_scripts/prod/data/earthstat/__init__.py similarity index 100% rename from geobricks_data_scripts/test/config/__init__.py rename to geobricks_data_scripts/prod/data/earthstat/__init__.py diff --git a/geobricks_data_scripts/test/data/__init__.py b/geobricks_data_scripts/prod/data/earthstat/delete/__init__.py similarity index 100% rename from geobricks_data_scripts/test/data/__init__.py rename to geobricks_data_scripts/prod/data/earthstat/delete/__init__.py diff --git a/geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py similarity index 100% rename from geobricks_data_scripts/test/data/earthstat/delete/delete_all_earthstat.py rename to geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py diff --git a/geobricks_data_scripts/test/data/earthstat/__init__.py b/geobricks_data_scripts/prod/data/earthstat/publish/__init__.py similarity index 100% rename from geobricks_data_scripts/test/data/earthstat/__init__.py rename to geobricks_data_scripts/prod/data/earthstat/publish/__init__.py diff --git a/geobricks_data_scripts/test/data/earthstat/publish/publish_harvest.py b/geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py similarity index 100% rename from geobricks_data_scripts/test/data/earthstat/publish/publish_harvest.py rename to geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py diff --git a/geobricks_data_scripts/test/data/earthstat/delete/__init__.py b/geobricks_data_scripts/prod/data/ghg/__init__.py similarity index 100% rename from geobricks_data_scripts/test/data/earthstat/delete/__init__.py rename to geobricks_data_scripts/prod/data/ghg/__init__.py diff --git a/geobricks_data_scripts/test/data/earthstat/publish/__init__.py b/geobricks_data_scripts/prod/data/ghg/delete/__init__.py similarity index 100% rename from geobricks_data_scripts/test/data/earthstat/publish/__init__.py rename to geobricks_data_scripts/prod/data/ghg/delete/__init__.py diff --git a/geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py similarity index 100% rename from geobricks_data_scripts/test/data/ghg/delete/data_manager_delete_all_example.py rename to geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py diff --git a/geobricks_data_scripts/test/data/ghg/__init__.py b/geobricks_data_scripts/prod/data/ghg/publish/__init__.py similarity index 100% rename from geobricks_data_scripts/test/data/ghg/__init__.py rename to geobricks_data_scripts/prod/data/ghg/publish/__init__.py diff --git a/geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data.py similarity index 100% rename from geobricks_data_scripts/test/data/ghg/publish/publish_GHG_data.py rename to geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data.py diff --git a/geobricks_data_scripts/test/utils/.gitignore b/geobricks_data_scripts/prod/utils/.gitignore similarity index 100% rename from geobricks_data_scripts/test/utils/.gitignore rename to geobricks_data_scripts/prod/utils/.gitignore diff --git a/geobricks_data_scripts/test/data/ghg/delete/__init__.py b/geobricks_data_scripts/prod/utils/__init__.py similarity index 100% rename from geobricks_data_scripts/test/data/ghg/delete/__init__.py rename to geobricks_data_scripts/prod/utils/__init__.py diff --git a/geobricks_data_scripts/test/utils/data_manager_util.py b/geobricks_data_scripts/prod/utils/data_manager_util.py similarity index 100% rename from geobricks_data_scripts/test/utils/data_manager_util.py rename to geobricks_data_scripts/prod/utils/data_manager_util.py diff --git a/geobricks_data_scripts/test/__init__.py b/geobricks_data_scripts/test/__init__.py deleted file mode 100644 index 232e348..0000000 --- a/geobricks_data_scripts/test/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -_package__ = 'geobricks_data_scripts' -__author__ = 'Barbaglia, Guido - Murzilli, Simone' -__email__ = 'guido.barbaglia@gmail.com; simone.murzilli@gmail.com;' -__license__ = 'GPL2' diff --git a/geobricks_data_scripts/test/coding_system/create_cs_csv.py b/geobricks_data_scripts/test/coding_system/create_cs_csv.py deleted file mode 100644 index 3ff0dde..0000000 --- a/geobricks_data_scripts/test/coding_system/create_cs_csv.py +++ /dev/null @@ -1,48 +0,0 @@ -import csv - -items = [ - "Cultivation Organic Soils", - "GFED4 Burned Areas", - "GFED4 Burned Areas - Barren or sparsely vegetated", - "GFED4 Burned Areas - Closed shrubland", - "GFED4 Burned Areas - Croplands", - "GFED4 Burned Areas - Deciduous Broadleaf forest", - "GFED4 Burned Areas - Deciduous Needleleaf forest", - "GFED4 Burned Areas - Evergreen Broadleaf forest", - "GFED4 Burned Areas - Evergreen Needleleaf forest", - "GFED4 Burned Areas - Grassland", - "GFED4 Burned Areas - Humid Tropical Forest", - "GFED4 Burned Areas - Mixed forest", - "GFED4 Burned Areas - Open shrubland", - "GFED4 Burned Areas - Organic soils", - "GFED4 Burned Areas - Other forest", - "GFED4 Burned Areas - Savanna", - "GFED4 Burned Areas - Unclassified", - "GFED4 Burned Areas - Urban and built-up", - "GFED4 Burned Areas - Woody savanna", - "Global Ecological Zones (GEZ) 2010", - "Global Land Cover 2000 (GLC2000)", - "Gridded Livestock of the World v. 2.01", - "Harmonized World Soil Database - Organic soils", - "JRC climate zone", - "JRC climate zone (4326)", - "MODIS - Land Cover Type UMD" -] - - -final = [] -with open('csv/cs_ghg.csv', 'wb') as csvfile: - w = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) - for i in items: - s = i.lower().replace(" ", "_") - w.writerow([s, i]) - final.append([s, i]) -print final - - - - - - - - diff --git a/geobricks_data_scripts/test/coding_system/csv/cs_ghg.csv b/geobricks_data_scripts/test/coding_system/csv/cs_ghg.csv deleted file mode 100644 index 708565a..0000000 --- a/geobricks_data_scripts/test/coding_system/csv/cs_ghg.csv +++ /dev/null @@ -1,25 +0,0 @@ -cultivation_organic_soils,Cultivation Organic Soils -gfed4_burned_areas,GFED4 Burned Areas -gfed4_burned_areas_-_barren_or_sparsely_vegetated,GFED4 Burned Areas - Barren or sparsely vegetated -gfed4_burned_areas_-_closed_shrubland,GFED4 Burned Areas - Closed shrubland -gfed4_burned_areas_-_croplands,GFED4 Burned Areas - Croplands -gfed4_burned_areas_-_deciduous_broadleaf_forest,GFED4 Burned Areas - Deciduous Broadleaf forest -gfed4_burned_areas_-_deciduous_needleleaf_forest,GFED4 Burned Areas - Deciduous Needleleaf forest -gfed4_burned_areas_-_evergreen_broadleaf_forest,GFED4 Burned Areas - Evergreen Broadleaf forest -gfed4_burned_areas_-_evergreen_needleleaf_forest,GFED4 Burned Areas - Evergreen Needleleaf forest -gfed4_burned_areas_-_grassland,GFED4 Burned Areas - Grassland -gfed4_burned_areas_-_humid_tropical_forest,GFED4 Burned Areas - Humid Tropical Forest -gfed4_burned_areas_-_mixed_forest,GFED4 Burned Areas - Mixed forest -gfed4_burned_areas_-_open_shrubland,GFED4 Burned Areas - Open shrubland -gfed4_burned_areas_-_organic_soils,GFED4 Burned Areas - Organic soils -gfed4_burned_areas_-_other_forest,GFED4 Burned Areas - Other forest -gfed4_burned_areas_-_savanna,GFED4 Burned Areas - Savanna -gfed4_burned_areas_-_unclassified,GFED4 Burned Areas - Unclassified -gfed4_burned_areas_-_urban_and_built-up,GFED4 Burned Areas - Urban and built-up -gfed4_burned_areas_-_woody_savanna,GFED4 Burned Areas - Woody savanna -global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 -global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) -gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 -harmonized_world_soil_database_-_organic_soils,Harmonized World Soil Database - Organic soils -jrc_climate_zone,JRC climate zone -modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD diff --git a/geobricks_data_scripts/test/coding_system/csv/cs_maghg.csv b/geobricks_data_scripts/test/coding_system/csv/cs_maghg.csv deleted file mode 100644 index 959a829..0000000 --- a/geobricks_data_scripts/test/coding_system/csv/cs_maghg.csv +++ /dev/null @@ -1,45 +0,0 @@ -dm_burning_grasslands,DM Burning Grasslands -ch4_emissions_burning_woodysavanna,CH4 Emissions Burning WoodySavanna -n2o_emissions_burning_grasslands,N2O Emissions Burning Grasslands -n2o_emissions_burning_woodysavanna,N2O Emissions Burning WoodySavanna -co2_emissions_burning_closedshrublands,CO2 Emissions Burning ClosedShrublands -co2_emissions_burning_openshrublands,CO2 Emissions Burning OpenShrublands -co2_emissions_burning_humidtropicalforests,CO2 Emissions Burning HumidTropicalForests -dm_burning_openshrublands,DM Burning OpenShrublands -n2o_emissions_burning_otherforests,N2O Emissions Burning OtherForests -n2o_emissions_burning_humidtropicalforests,N2O Emissions Burning HumidTropicalForests -n2o_emissions_burning_savanna,N2O Emissions Burning Savanna -ch4_emissions_burning_humidtropicalforests,CH4 Emissions Burning HumidTropicalForests -ch4_emissions_burning_peatlands,CH4 Emissions Burning Peatlands -n2o_emissions_burning_peatlands,N2O Emissions Burning Peatlands -ch4_emissions_burning_closedshrublands,CH4 Emissions Burning ClosedShrublands -co2_emissions_burning_grasslands,CO2 Emissions Burning Grasslands -ch4_emissions_burning_grasslands,CH4 Emissions Burning Grasslands -openshrublands_fuelbiomass,OpenShrublands FuelBiomass -co2_emissions_burning_woodysavanna,CO2 Emissions Burning WoodySavanna -ch4_emissions_burning_otherforests,CH4 Emissions Burning OtherForests -dm_burning_closedshrublands,DM Burning ClosedShrublands -dm_burning_woodysavanna,DM Burning WoodySavanna -ch4_emissions_burning_savanna,CH4 Emissions Burning Savanna -co2_emissions_burning_otherforests,CO2 Emissions Burning OtherForests -n2o_emissions_burning_closedshrublands,N2O Emissions Burning ClosedShrublands -co2_emissions_burning_peatlands,CO2 Emissions Burning Peatlands -co2_emissions_burning_savanna,CO2 Emissions Burning Savanna -dm_burning_peatlands,DM Burning Peatlands -dm_burning_humidtropicalforests,DM Burning HumidTropicalForests -savanna_fuelbiomass,Savanna FuelBiomass -n2o_emissions_burning_openshrublands,N2O Emissions Burning OpenShrublands -tropical_forests_ch4_ef,Tropical Forests CH4 EF -dm_burning_savanna,DM Burning Savanna -ch4_emissions_burning_openshrublands,CH4 Emissions Burning OpenShrublands -dm_burning_otherforests,DM Burning OtherForests -tropical_forests_fuelbiomass,Tropical Forests FuelBiomass -woodysavanna_fuelbiomass,WoodySavanna FuelBiomass -grasslands_fuelbiomass,Grasslands FuelBiomass -nontropical_forests_fuelbiomass,NonTropical Forests FuelBiomass -tropical_forests_co2_ef,Tropical Forests CO2 EF -extratropical_forests_n2o_ef,ExtraTropical Forests N2O EF -closedshrublands_fuelbiomass,ClosedShrublands FuelBiomass -tropical_forests_n2o_ef,Tropical Forests N2O EF -extratropical_forests_co2_ef,ExtraTropical Forests CO2 EF -extratropical_forests_ch4_ef,ExtraTropical Forests CH4 EF diff --git a/geobricks_data_scripts/test/coding_system/csv/final.csv b/geobricks_data_scripts/test/coding_system/csv/final.csv deleted file mode 100644 index 90c32c6..0000000 --- a/geobricks_data_scripts/test/coding_system/csv/final.csv +++ /dev/null @@ -1,23 +0,0 @@ -cultivation_organic_soils_-_croplands,Cultivation Organic Soils - Croplands -burned_areas_-_barren_or_sparsely_vegetated,Burned Areas - Barren or sparsely vegetated -burned_areas_-_closed_shrubland,Burned Areas - Closed shrubland -burned_areas_-_croplands,Burned Areas - Croplands -burned_areas_-_deciduous_broadleaf_forest,Burned Areas - Deciduous Broadleaf forest -burned_areas_-_deciduous_needleleaf_forest,Burned Areas - Deciduous Needleleaf forest -burned_areas_-_evergreen_broadleaf_forest,Burned Areas - Evergreen Broadleaf forest -burned_areas_-_evergreen_needleleaf_forest,Burned Areas - Evergreen Needleleaf forest -burned_areas_-_grassland,Burned Areas - Grassland -burned_areas_-_humid_tropical_forest,Burned Areas - Humid Tropical Forest -burned_areas_-_mixed_forest,Burned Areas - Mixed forest -burned_areas_-_open_shrubland,Burned Areas - Open shrubland -burned_areas_-_organic_soils,Burned Areas - Organic soils -burned_areas_-_other_forest,Burned Areas - Other forest -burned_areas_-_savanna,Burned Areas - Savanna -burned_areas_-_unclassified,Burned Areas - Unclassified -burned_areas_-_urban_and_built-up,Burned Areas - Urban and built-up -burned_areas_-_woody_savanna,Burned Areas - Woody savanna -global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 -global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) -gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 -jrc_climate_zone,JRC climate zone -modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD \ No newline at end of file diff --git a/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv b/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv deleted file mode 100644 index 403530d..0000000 --- a/geobricks_data_scripts/test/coding_system/csv/final_without_maghg_and_soil.csv +++ /dev/null @@ -1,23 +0,0 @@ -cultivation_organic_soils_-_croplands,Cultivation Organic Soils - Croplands -burned_areas_-_barren_or_sparsely_vegetated,Burned Areas - Barren or sparsely vegetated -burned_areas_-_closed_shrubland,Burned Areas - Closed shrubland -burned_areas_-_croplands,Burned Areas - Croplands -burned_areas_-_deciduous_broadleaf_forest,Burned Areas - Deciduous Broadleaf forest -burned_areas_-_deciduous_needleleaf_forest,Burned Areas - Deciduous Needleleaf forest -burned_areas_-_evergreen_broadleaf_forest,Burned Areas - Evergreen Broadleaf forest -burned_areas_-_evergreen_needleleaf_forest,Burned Areas - Evergreen Needleleaf forest -burned_areas_-_grassland,Burned Areas - Grassland -burned_areas_-_humid_tropical_forest,Burned Areas - Humid Tropical Forest -burned_areas_-_mixed_forest,Burned Areas - Mixed forest -burned_areas_-_open_shrubland,Burned Areas - Open shrubland -burned_areas_-_organic_soils,Burned Areas - Organic soils -burned_areas_-_other_forest,Burned Areas - Other forest -burned_areas_-_savanna,Burned Areas - Savanna -burned_areas_-_unclassified,Burned Areas - Unclassified -burned_areas_-_urban_and_built-up,Burned Areas - Urban and built-up -burned_areas_-_woody_savanna,Burned Areas - Woody savanna -global_ecological_zones_(gez)_2010,Global Ecological Zones (GEZ) 2010 -global_land_cover_2000_(glc2000),Global Land Cover 2000 (GLC2000) -gridded_livestock_of_the_world_v_201,Gridded Livestock of the World v2.01 -jrc_climate_zone,JRC climate zone -modis_-_land_cover_type_umd,MODIS - Land Cover Type UMD diff --git a/geobricks_data_scripts/test/coding_system/publish_final_cs.py b/geobricks_data_scripts/test/coding_system/publish_final_cs.py deleted file mode 100644 index 8852627..0000000 --- a/geobricks_data_scripts/test/coding_system/publish_final_cs.py +++ /dev/null @@ -1,43 +0,0 @@ -import csv -import json - -from geobricks_metadata_manager.core.metadata_manager_d3s_core import MetadataManager -from geobricks_data_scripts.test.config.config import settings - -metadata_manager = MetadataManager(settings) - -print metadata_manager - -cs = { - "metadata": { - "uid": "layer_products", - "version": "1.0", - "title": { - "EN": "Layers" - }, - "meContent": { - "resourceRepresentationType": "codelist" - } - }, - "data": [ - - ] -} - - -with open('csv/final.csv', 'rb') as csvfile: - rows = csv.reader(csvfile) - for row in rows: - cs["data"].append({ - "code": row[0], - "title": { - "EN": row[1] - } - }) - - -print json.dumps(cs) -print metadata_manager.publish_coding_system(cs, True) - - - diff --git a/geobricks_data_scripts/test/coding_system/publish_maghg_cs.py b/geobricks_data_scripts/test/coding_system/publish_maghg_cs.py deleted file mode 100644 index f4a93fb..0000000 --- a/geobricks_data_scripts/test/coding_system/publish_maghg_cs.py +++ /dev/null @@ -1,26 +0,0 @@ -import glob -import os -import csv - -files = glob.glob("/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/*.tif") - -cached_codes = [] -codes = [] -for f in files: - if "_3857" in f: - if "20" in f or "19" in f: - # it contains a date - folder, filename = os.path.split(f.rsplit("_", 2)[0]) - else: - folder, filename = os.path.split(f.rsplit("_", 1)[0]) - if filename not in cached_codes: - cached_codes.append(filename) - codes.append([filename.lower().replace(" ","_"), filename.replace("_", " ")]) - - -with open('csv/cs_maghg.csv', 'wb') as csvfile: - w = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) - for code in codes: - w.writerow(code) - - diff --git a/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py b/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py deleted file mode 100644 index e13438a..0000000 --- a/geobricks_data_scripts/test/common_data/coding_system/update_mapProjections.py +++ /dev/null @@ -1,33 +0,0 @@ -from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager - -data_manager = get_data_manager() - -cs = { - "metadata": { - "uid": "mapProjections", - "version": "1.0" - }, - "data": [ - { - "code": "EPSG:4326", - "title": { - "EN": "Latlong" - } - }, - { - "code": "EPSG:3857", - "title": { - "EN": "Spherical Mercator projection" - } - }, - { - "code": "EPSG:900913", - "title": { - "EN": "Equivalent to Spherical Mercator Projection" - } - } - ] -} - -# Update coding system -data_manager.metadata_manager.update_coding_system(cs, True) \ No newline at end of file diff --git a/geobricks_data_scripts/test/data/ghg/publish/__init__.py b/geobricks_data_scripts/test/data/ghg/publish/__init__.py deleted file mode 100644 index 22d574d..0000000 --- a/geobricks_data_scripts/test/data/ghg/publish/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'vortex' diff --git a/geobricks_data_scripts/test/utils/__init__.py b/geobricks_data_scripts/test/utils/__init__.py deleted file mode 100644 index 22d574d..0000000 --- a/geobricks_data_scripts/test/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'vortex' From 442b07f2a603399ca5a280fb05776ec148f22adc Mon Sep 17 00:00:00 2001 From: simonefenix Date: Fri, 13 Feb 2015 11:23:15 +0100 Subject: [PATCH 16/39] updated prod --- geobricks_data_scripts/README.md | 27 +++++++++++++++++++ .../delete/data_manager_delete_all_example.py | 2 +- geobricks_data_scripts/prod/README | 1 + .../check_consistency/check_consistency.py | 8 +++--- geobricks_data_scripts/prod/config/.gitignore | 2 ++ .../earthstat/delete/delete_all_earthstat.py | 7 ++--- .../data/earthstat/publish/publish_harvest.py | 2 +- .../delete/data_manager_delete_all_example.py | 4 +-- .../prod/data/ghg/publish/publish_GHG_data.py | 2 +- .../prod/utils/data_manager_util.py | 2 +- 10 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 geobricks_data_scripts/prod/README diff --git a/geobricks_data_scripts/README.md b/geobricks_data_scripts/README.md index e69de29..499a99a 100644 --- a/geobricks_data_scripts/README.md +++ b/geobricks_data_scripts/README.md @@ -0,0 +1,27 @@ +# Structure of geobricks_data_scripts + +## geobricks_data_scripts.dev + +It's used on localhost (geoserver and d3s). + +**IMPORTANT**: It's used to **RENAME**, **MANAGE (i.e. move folders)** and **PROCESSING** the layers + +## geobricks_data_scripts.demo + +It's used for demo purposes and the data are **ALREADY BEEN PROCESSED**. It's used just to publish, delete and manage data. + +### geobricks_data_scripts.demo.fenix + +It uses: + +* D3S: fenix.fao.org/d3s +* GEOSERVER (master): lprapp16.fao.org:20200/geoserver/rest + + +### geobricks_data_scripts.demo.old_ghg + +**@Deprecated** + +## PROD + +**TODO** diff --git a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py index 934a485..b58c736 100644 --- a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py @@ -6,7 +6,7 @@ for layer in layers: try: print layer["uid"] - data_manager.delete(layer["uid"]) + #data_manager.delete(layer["uid"]) except Exception, e: print e pass \ No newline at end of file diff --git a/geobricks_data_scripts/prod/README b/geobricks_data_scripts/prod/README new file mode 100644 index 0000000..bf794c3 --- /dev/null +++ b/geobricks_data_scripts/prod/README @@ -0,0 +1 @@ +DA CAMBIARE LA CONFIGURAZIONE!! \ No newline at end of file diff --git a/geobricks_data_scripts/prod/check_consistency/check_consistency.py b/geobricks_data_scripts/prod/check_consistency/check_consistency.py index a3a5c44..f927a9b 100644 --- a/geobricks_data_scripts/prod/check_consistency/check_consistency.py +++ b/geobricks_data_scripts/prod/check_consistency/check_consistency.py @@ -1,11 +1,11 @@ -import os -import glob from geobricks_common.core.log import logger from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager -from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder log = logger(__file__) data_manager = get_data_manager() -data_manager.check_consistency() +result = data_manager.check_consistency() + +for r in result["storage"]: + log.info(r) diff --git a/geobricks_data_scripts/prod/config/.gitignore b/geobricks_data_scripts/prod/config/.gitignore index 0d20b64..d5ae511 100644 --- a/geobricks_data_scripts/prod/config/.gitignore +++ b/geobricks_data_scripts/prod/config/.gitignore @@ -1 +1,3 @@ *.pyc +config.py +*.py diff --git a/geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py index 8afa556..9571d55 100644 --- a/geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py +++ b/geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py @@ -1,11 +1,12 @@ -from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager data_manager = get_data_manager() layers = data_manager.get_all_layers() for layer in layers: try: - if "earthstat" in layer["dsd"]["workspace"]: - data_manager.delete(layer["uid"]) + print layer["uid"] + #if "earthstat" in layer["dsd"]["workspace"]: + # data_manager.delete(layer["uid"]) except Exception, e: print "Error", e pass \ No newline at end of file diff --git a/geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py b/geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py index 55e0bfd..3758be7 100644 --- a/geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py +++ b/geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py @@ -1,6 +1,6 @@ import glob import os -from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder diff --git a/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py index 80723bc..0dfd6a9 100644 --- a/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py @@ -1,11 +1,11 @@ -from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager data_manager = get_data_manager() layers = data_manager.get_all_layers() for layer in layers: try: print layer["uid"] - data_manager.delete(layer["uid"]) + #data_manager.delete(layer["uid"]) except Exception, e: print e pass \ No newline at end of file diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data.py index 49b9b1c..f337628 100644 --- a/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data.py +++ b/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data.py @@ -1,7 +1,7 @@ import os import glob from geobricks_common.core.log import logger -from geobricks_data_scripts.test.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder log = logger(__file__) diff --git a/geobricks_data_scripts/prod/utils/data_manager_util.py b/geobricks_data_scripts/prod/utils/data_manager_util.py index 2acda6e..695fc89 100644 --- a/geobricks_data_scripts/prod/utils/data_manager_util.py +++ b/geobricks_data_scripts/prod/utils/data_manager_util.py @@ -1,5 +1,5 @@ from geobricks_data_manager.core.data_manager_core import DataManager -from geobricks_data_scripts.test.config.config import settings +from geobricks_data_scripts.prod.config.config import settings # get the default data manager for demo db def get_data_manager(): From a65104bc1b1d4a40df67f4d35f869b4355bb70cc Mon Sep 17 00:00:00 2001 From: simonefenix Date: Fri, 13 Feb 2015 11:23:27 +0100 Subject: [PATCH 17/39] updated prod --- geobricks_data_scripts/prod/config/config.py | 77 -------------------- 1 file changed, 77 deletions(-) delete mode 100644 geobricks_data_scripts/prod/config/config.py diff --git a/geobricks_data_scripts/prod/config/config.py b/geobricks_data_scripts/prod/config/config.py deleted file mode 100644 index 0e62287..0000000 --- a/geobricks_data_scripts/prod/config/config.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging -import os -import json - -settings = { - - "settings": { - - - # To be used by Flask: DEVELOPMENT ONLY - "debug": True, - - # Flask host: DEVELOPMENT ONLY - "host": "localhost", - - # Flask port: DEVELOPMENT ONLY - "port": 5555, - - # Logging configurations - "logging": { - "level": logging.ERROR, - "format": "%(asctime)s | %(levelname)-8s | %(name)-20s | Line: %(lineno)-5d | %(message)s", - "datefmt": "%d-%m-%Y | %H:%M:%s" - }, - - # Email configurations - "email": { - "settings": "path_to_the_email.json", - "user": None, - "password": None - }, - - # Folders - "folders": { - "config": "config/", - "tmp": "/home/vortex/Desktop/LAYERS/geobricks/tmp/", - "geoserver_datadir": "/home/vortex/programs/SERVERS/tomcat_geoservers/geoserver_data_2_5_3/data/", - "distribution": "/home/vortex/Desktop/LAYERS/geobricks/distribution/", - "storage": "/home/vortex/Desktop/LAYERS/geobricks/storage/" - }, - - # Geoserver - "geoserver": { - "geoserver_master": "http://fenix.fao.org:20200/geoserver/rest", - "geoserver_slaves": [], - "username": "fenix", - "password": "Fenix2014", - }, - - # Metadata - "metadata": { - "url_create_metadata": "http://fenix.fao.org/d3s/msd/resources/metadata", - "url_get_metadata_uid": "http://fenix.fao.org/d3s/msd/resources/metadata/uid/", - - # delete metadata - "url_delete_metadata": "http://fenix.fao.org/d3s/msd/resources/metadata/uid/", - - # get metadata - "url_get_metadata": "http://fenix.fao.org/d3s/msd/resources/find", - "url_get_metadata_full": "http://fenix.fao.org/d3s/msd/resources/metadata/uid/?full=true&dsd=true", - - # coding system - "url_create_coding_system": "http://fenix.fao.org/d3s/msd/resources", - "url_data_coding_system": "http://fenix.fao.org/d3s/msd/resources/data/uid/", - - # DSD - "url_overwrite_dsd_rid": "http://fenix.fao.org/d3s/msd/resources/dsd" - } - } -} - - -# Setting email adderess from configuration file -def set_email_settings(): - if os.path.isfile(settings["settings"]["email"]["settings"]): - settings["settings"]["email"] = json.loads(open(settings["settings"]["email"]["settings"]).read()) -set_email_settings() \ No newline at end of file From 23631174bb59f922dc11a4d25d5c6cd4418b97a6 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Fri, 13 Feb 2015 11:34:08 +0100 Subject: [PATCH 18/39] TODO --- .../utils/harvest/publish_harvest.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/geobricks_data_scripts/utils/harvest/publish_harvest.py b/geobricks_data_scripts/utils/harvest/publish_harvest.py index ef9c1fc..cad1405 100644 --- a/geobricks_data_scripts/utils/harvest/publish_harvest.py +++ b/geobricks_data_scripts/utils/harvest/publish_harvest.py @@ -37,18 +37,20 @@ def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, p log.info(metadata) # check epsg if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": + # TODO: get code update_dsd_layer_to_distribution_layer(data_manager, metadata) - # link 4326 layer to 3857 visualization layer + # link 4326 (or other Proj EPSG) layer to 3857 visualization layer for metadata in metadatas: log.info(metadata) # check epsg if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": + # TODO: get code update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace) - return metadatas + def update_folder_style(data_manager, folder, style=None): ''' Updates all layer style in the folder. @@ -72,14 +74,16 @@ def update_folder_style(data_manager, folder, style=None): -def update_dsd_layer_to_distribution_layer(data_manager, metadata): +def update_dsd_layer_to_distribution_layer(data_manager, metadata, epsg_code): log.info(metadata) - update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], "4326", "distribution") + update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], epsg_code, "distribution") + #update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], "4326", "distribution") -def update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace): +def update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace, epsg_code): log.info(metadata) - update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], "3857", "visualization", workspace) + update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], epsg_code, "visualization", workspace) + #update_dsd_layer_with_layer_dist_or_vis(data_manager, metadata["uid"], "3857", "visualization", workspace) def publish_geoserver(data_manager, path, metadata): From 9bcaebdc54841ba6eb090ef545ebca8b8dc09b04 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Wed, 4 Mar 2015 14:32:52 +0100 Subject: [PATCH 19/39] fixed processing issue --- .../processing/processing_earthstat_data.py | 41 +++++++++++++------ .../data/ghg/process/process_burned_areas.py | 17 ++++---- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py b/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py index 0971cdf..9e71ac0 100644 --- a/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py +++ b/geobricks_data_scripts/dev/data/earthstat/processing/processing_earthstat_data.py @@ -1,6 +1,7 @@ import glob import os import shutil +import copy from geobricks_common.core.log import logger from geobricks_common.core.filesystem import get_filename from geobricks_processing.core.processing_core import process_obj @@ -119,11 +120,15 @@ def process(src_folder, dest_folder="", folder_type="geoserver", prj="3857", ove # dest_folder = create_output_folder(folder, folder_type, overwrite) files = glob.glob(os.path.join(folder, "*." + file_type)) for f in files: + print f + if "area" in f: output_path = os.path.join(dest_folder, "earthstat_crop_area") elif "yield" in f: output_path = os.path.join(dest_folder, "earthstat_crop_yield") + + output_path = create_output_folder(output_path, folder_type, overwrite) filename = get_filename(f) @@ -132,22 +137,32 @@ def process(src_folder, dest_folder="", folder_type="geoserver", prj="3857", ove output_file_name = filename + "_" + prj + ".geotiff" # TODO: switch between processes print "Processing:", output_file_name + if prj == "3857": + process_steps = copy.deepcopy(process_obj_3857) # TODO: worksaround for the various sourcepath e dstpath - process_obj_3857[0]["source_path"] = [f] - process_obj_3857[1]["output_path"] = output_path - process_obj_3857[1]["output_file_name"] = output_file_name - for p in process_obj_3857: + process_steps[0]["source_path"] = [f] + process_steps[1]["output_path"] = output_path + process_steps[1]["output_file_name"] = output_file_name + for p in process_steps: + print "-----------" + print p p["source_path"] = p["source_path"] if "source_path" in p else result + print "source_path:", p["source_path"] + if "output_path" in p: print "output_path:", p["output_path"] + if "output_file_name" in p: print "output_file_name:", p["output_file_name"] result = process_obj(p) elif prj == "4326": - process_obj_4326[0]["source_path"] = [f] - process_obj_4326[1]["output_path"] = output_path - process_obj_4326[1]["output_file_name"] = output_file_name - for p in process_obj_4326: + process_steps = copy.deepcopy(process_obj_4326) + process_steps[0]["source_path"] = [f] + process_steps[1]["output_path"] = output_path + process_steps[1]["output_file_name"] = output_file_name + for p in process_steps: p["source_path"] = p["source_path"] if "source_path" in p else result result = process_obj(p) + # break + # break def create_output_folder(folder, folder_type="geoserver", overwrite=False): dest_folder = os.path.join(folder, folder_type) @@ -190,20 +205,20 @@ def move_file_to_publish(src_folder, to_publish_folder, file_type="geotiff"): # load scripts if __name__ == '__main__': - src_folder = "/home/vortex/Desktop/LAYERS/earthstat/175CropsYieldArea_geotiff/" + src_folder = "/home/vortex/Desktop/LAYERS/earthstat/175CropsYieldArea_geotiff/" dest_folder = "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/" - # process(src_folder, dest_folder, "geoserver", "3857") - #process(src_folder, dest_folder, "storage", "4326") + #process(src_folder, dest_folder, "geoserver", "3857") + process(src_folder, dest_folder, "storage", "4326") # MOVE TO PUBLISH FOLDER to_publish_folder_crop_area = "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area" to_publish_folder_crop_yield = "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_yield" #move_file_to_publish(dest_folder + "earthstat_crop_area/geoserver/", to_publish_folder_crop_area) - #move_file_to_publish(dest_folder + "earthstat_crop_area/storage/", to_publish_folder_crop_area) + move_file_to_publish(dest_folder + "earthstat_crop_area/storage/", to_publish_folder_crop_area) #move_file_to_publish(dest_folder + "earthstat_crop_yield/geoserver/", to_publish_folder_crop_yield) - #move_file_to_publish(dest_folder + "earthstat_crop_yield/storage/", to_publish_folder_crop_yield) + move_file_to_publish(dest_folder + "earthstat_crop_yield/storage/", to_publish_folder_crop_yield) # This is used to create Storage folders to move to the server (INTHEORY IT'S NOT NEEDED) diff --git a/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py b/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py index 0f885f2..2b71754 100644 --- a/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py +++ b/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py @@ -1,6 +1,7 @@ import glob import os import shutil +import copy from geobricks_common.core.log import logger from geobricks_common.core.filesystem import get_filename from geobricks_processing.core.processing_core import process_obj @@ -81,21 +82,23 @@ def process(src_folder, dest_folder, file_type): # TODO: switch between processes if prj == "3857": + process_steps = copy.deepcopy(process_obj_3857) output_file_name = filename + ".geotiff" print "Processing:", output_file_name # TODO: worksaround for the various sourcepath e dstpath - process_obj_3857[0]["source_path"] = [f] - process_obj_3857[0]["output_path"] = output_path - process_obj_3857[0]["output_file_name"] = output_file_name - for p in process_obj_3857: + process_steps[0]["source_path"] = [f] + process_steps[0]["output_path"] = output_path + process_steps[0]["output_file_name"] = output_file_name + for p in process_step: p["source_path"] = p["source_path"] if "source_path" in p else result result = process_obj(p) elif prj == "4326": + process_steps = copy.deepcopy(process_obj_4326) output_file_name = filename + "_" + prj + ".geotiff" print "Processing:", output_file_name - process_obj_4326[0]["source_path"] = [f] - process_obj_4326[0]["output_path"] = output_path - process_obj_4326[0]["output_file_name"] = output_file_name + process_steps[0]["source_path"] = [f] + process_steps[0]["output_path"] = output_path + process_steps[0]["output_file_name"] = output_file_name for p in process_obj_4326: p["source_path"] = p["source_path"] if "source_path" in p else result result = process_obj(p) From 0d450135c894004cd9ee4e5242f1cfddc6a20a16 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Fri, 27 Mar 2015 17:13:13 +0100 Subject: [PATCH 20/39] updated data_script --- geobricks_data_scripts/dev/config/config.py | 18 +-- .../data/earthstat/publish/publish_harvest.py | 2 +- .../dev/data/ghg/{delete => old}/__init__.py | 0 .../ghg/{process => old/delete}/__init__.py | 0 .../delete/data_manager_delete_all_example.py | 2 +- .../ghg/{ => old}/delete/delete_a_layer.py | 0 .../process}/__init__.py | 0 .../{ => old}/process/process_burned_areas.py | 0 .../process_folders}/__init__.py | 0 .../rename_burned_areas_folders_and_files.py | 0 .../publish/__init__.py} | 0 .../ghg/{ => old}/publish/publish_GHG_data.py | 0 .../ghg/{ => old}/publish/publish_harvest.py | 0 .../dev/data/ghg/old/update/__init__.py | 1 + .../dev/data/ghg/old/update/style_update.py | 1 + .../dev/data/ghg/publish/publish_data.py | 128 ++++++++++++++++++ .../utils/harvest/harvest_rasters.py | 2 +- .../utils/harvest/publish_harvest.py | 3 +- .../utils/metadata/metadata.py | 3 +- 19 files changed, 145 insertions(+), 15 deletions(-) rename geobricks_data_scripts/dev/data/ghg/{delete => old}/__init__.py (100%) rename geobricks_data_scripts/dev/data/ghg/{process => old/delete}/__init__.py (100%) rename geobricks_data_scripts/dev/data/ghg/{ => old}/delete/data_manager_delete_all_example.py (86%) rename geobricks_data_scripts/dev/data/ghg/{ => old}/delete/delete_a_layer.py (100%) rename geobricks_data_scripts/dev/data/ghg/{process_folders => old/process}/__init__.py (100%) rename geobricks_data_scripts/dev/data/ghg/{ => old}/process/process_burned_areas.py (100%) rename geobricks_data_scripts/dev/data/ghg/{update => old/process_folders}/__init__.py (100%) rename geobricks_data_scripts/dev/data/ghg/{ => old}/process_folders/rename_burned_areas_folders_and_files.py (100%) rename geobricks_data_scripts/dev/data/ghg/{update/style_update.py => old/publish/__init__.py} (100%) rename geobricks_data_scripts/dev/data/ghg/{ => old}/publish/publish_GHG_data.py (100%) rename geobricks_data_scripts/dev/data/ghg/{ => old}/publish/publish_harvest.py (100%) create mode 100644 geobricks_data_scripts/dev/data/ghg/old/update/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/old/update/style_update.py create mode 100644 geobricks_data_scripts/dev/data/ghg/publish/publish_data.py diff --git a/geobricks_data_scripts/dev/config/config.py b/geobricks_data_scripts/dev/config/config.py index 900ef04..0f645b4 100644 --- a/geobricks_data_scripts/dev/config/config.py +++ b/geobricks_data_scripts/dev/config/config.py @@ -41,7 +41,7 @@ # Geoserver "geoserver": { - "geoserver_master": "http://localhost:9090/geoserver/rest", + "geoserver_master": "http://localhost:10000/geoserver/rest", "geoserver_slaves": [], "username": "admin", "password": "geoserver", @@ -49,22 +49,22 @@ # Metadata "metadata": { - "url_create_metadata": "http://localhost:7788/v2/msd/resources/metadata", - "url_get_metadata_uid": "http://localhost:7788/v2/msd/resources/metadata/uid/", + "url_create_metadata": "http://localhost:7733/v2/msd/resources/metadata", + "url_get_metadata_uid": "http://localhost:7733/v2/msd/resources/metadata/uid/", # delete metadata - "url_delete_metadata": "http://localhost:7788/v2/msd/resources/metadata/uid/", + "url_delete_metadata": "http://localhost:7733/v2/msd/resources/metadata/uid/", # get metadata - "url_get_metadata": "http://localhost:7788/v2/msd/resources/find", - "url_get_metadata_full": "http://localhost:7788/v2/msd/resources/metadata/uid/?full=true&dsd=true", + "url_get_metadata": "http://localhost:7733/v2/msd/resources/find", + "url_get_metadata_full": "http://localhost:7733/v2/msd/resources/metadata/uid/?full=true&dsd=true", # coding system - "url_create_coding_system": "http://localhost:7788/v2/msd/resources", - "url_data_coding_system": "http://localhost:7788/v2/msd/resources/data/uid/", + "url_create_coding_system": "http://localhost:7733/v2/msd/resources", + "url_data_coding_system": "http://localhost:7733/v2/msd/resources/data/uid/", # DSD - "url_overwrite_dsd_rid": "http://localhost:7788/v2/msd/resources/dsd" + "url_overwrite_dsd_rid": "http://localhost:7733/v2/msd/resources/dsd" } } } diff --git a/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py b/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py index b18cd2d..c5ac44f 100644 --- a/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py +++ b/geobricks_data_scripts/dev/data/earthstat/publish/publish_harvest.py @@ -9,6 +9,6 @@ lang = "EN" workspace = "earthstat" -metadatas = harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area/", workspace, False, False, False, None) +metadatas = harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/TO_PUBLISH/earthstat_crop_area/", workspace, True, True, True, None) print metadatas # harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/", workspace) \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/delete/__init__.py b/geobricks_data_scripts/dev/data/ghg/old/__init__.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/delete/__init__.py rename to geobricks_data_scripts/dev/data/ghg/old/__init__.py diff --git a/geobricks_data_scripts/dev/data/ghg/process/__init__.py b/geobricks_data_scripts/dev/data/ghg/old/delete/__init__.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/process/__init__.py rename to geobricks_data_scripts/dev/data/ghg/old/delete/__init__.py diff --git a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/dev/data/ghg/old/delete/data_manager_delete_all_example.py similarity index 86% rename from geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py rename to geobricks_data_scripts/dev/data/ghg/old/delete/data_manager_delete_all_example.py index b58c736..934a485 100644 --- a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/dev/data/ghg/old/delete/data_manager_delete_all_example.py @@ -6,7 +6,7 @@ for layer in layers: try: print layer["uid"] - #data_manager.delete(layer["uid"]) + data_manager.delete(layer["uid"]) except Exception, e: print e pass \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py b/geobricks_data_scripts/dev/data/ghg/old/delete/delete_a_layer.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py rename to geobricks_data_scripts/dev/data/ghg/old/delete/delete_a_layer.py diff --git a/geobricks_data_scripts/dev/data/ghg/process_folders/__init__.py b/geobricks_data_scripts/dev/data/ghg/old/process/__init__.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/process_folders/__init__.py rename to geobricks_data_scripts/dev/data/ghg/old/process/__init__.py diff --git a/geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py b/geobricks_data_scripts/dev/data/ghg/old/process/process_burned_areas.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/process/process_burned_areas.py rename to geobricks_data_scripts/dev/data/ghg/old/process/process_burned_areas.py diff --git a/geobricks_data_scripts/dev/data/ghg/update/__init__.py b/geobricks_data_scripts/dev/data/ghg/old/process_folders/__init__.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/update/__init__.py rename to geobricks_data_scripts/dev/data/ghg/old/process_folders/__init__.py diff --git a/geobricks_data_scripts/dev/data/ghg/process_folders/rename_burned_areas_folders_and_files.py b/geobricks_data_scripts/dev/data/ghg/old/process_folders/rename_burned_areas_folders_and_files.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/process_folders/rename_burned_areas_folders_and_files.py rename to geobricks_data_scripts/dev/data/ghg/old/process_folders/rename_burned_areas_folders_and_files.py diff --git a/geobricks_data_scripts/dev/data/ghg/update/style_update.py b/geobricks_data_scripts/dev/data/ghg/old/publish/__init__.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/update/style_update.py rename to geobricks_data_scripts/dev/data/ghg/old/publish/__init__.py diff --git a/geobricks_data_scripts/dev/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/dev/data/ghg/old/publish/publish_GHG_data.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/publish/publish_GHG_data.py rename to geobricks_data_scripts/dev/data/ghg/old/publish/publish_GHG_data.py diff --git a/geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py b/geobricks_data_scripts/dev/data/ghg/old/publish/publish_harvest.py similarity index 100% rename from geobricks_data_scripts/dev/data/ghg/publish/publish_harvest.py rename to geobricks_data_scripts/dev/data/ghg/old/publish/publish_harvest.py diff --git a/geobricks_data_scripts/dev/data/ghg/old/update/__init__.py b/geobricks_data_scripts/dev/data/ghg/old/update/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/old/update/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/old/update/style_update.py b/geobricks_data_scripts/dev/data/ghg/old/update/style_update.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/old/update/style_update.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py b/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py new file mode 100644 index 0000000..45a2ca3 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py @@ -0,0 +1,128 @@ +import glob +import os +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder + + +data_manager = get_data_manager() + +lang = "EN" +workspace = "ghg" + +def get_maghg_defaultstyle_name(filename): + sldname = "" + if "FUELBIOMASS" in filename.upper(): + sldname = "fuelbiomass" + elif "CH4_EF" in filename.upper(): + sldname = "ch4_ef" + elif "CO2_EF" in filename.upper(): + sldname = "co2_ef" + elif "N2O_EF" in filename.upper(): + sldname = "n2o_ef" + elif "CH4" in filename.upper(): + sldname = "ch4" + elif "CO2" in filename.upper(): + sldname = "co2" + elif "DM" in filename.upper(): + sldname = "dm" + elif "N2O" in filename.upper(): + sldname = "n2o" + return sldname + "_emissions_burning_" + lang + +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/Cultivation_organic_soils_croplands/Cultivation_organic_soils_-_croplands/", workspace) +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GriddedLivestock/gridded_livestock_of_the_world_v_201/", workspace) +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/JRC_CLIMATE_ZONE/JRC_climate_zone/", workspace) +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GLC2000/glc2000_v1/", workspace) +# harvest_folder(data_manager, "/home/vortex/Deskt/op/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GEZ/global_ecological_zones_GEZ/", workspace) + + +def harvest_burned_areas(src_folder, output_folder="output", ): + print "TODO:" + +# GHG-MAG +# 4326 +def harvest_maghg_4326(): + maghg_4326_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/storage" + folders = glob.glob(os.path.join(maghg_4326_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + harvest_folder(data_manager, folder, workspace) + +# 3857 +def harvest_maghg_3857(): + maghg_3857_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/geoserver" + folders = glob.glob(os.path.join(maghg_3857_folder, "*")) + for folder in folders: + if os.path.isdir(folder): + harvest_folder(data_manager, folder, workspace) + + +def harvest_maghg_3857_updated_defaultstyle(): + maghg_3857_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/geoserver" + folders = glob.glob(os.path.join(maghg_3857_folder, "*")) + all_metadatas = [] + for folder in folders: + if os.path.isdir(folder): + all_metadatas.append(harvest_folder(data_manager, folder, workspace, False, False, False)) + + # update the metadata retrieved + for metadatas in all_metadatas: + for metadata in metadatas: + try: + print metadata["dsd"]["layerName"] + default_style = get_maghg_defaultstyle_name(metadata["dsd"]["layerName"]) + data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], default_style) + except Exception, e: + print "ERRRROOOOO!!!!!" + print e + + + # def create_storage_json_metadata(src_folder, code, product_codelist_id="layers_products"): + # for glob=glob + # + # + # def create_product_json(code, product_codelist_id="layers_products"): + # metadata = { + # "meContent" : { + # "seCoverage" : { + # "coverageSectors" : { + # "version" : "1.0", + # "codes" : [ { + # "code" : code + # } ], + # "idCodeList" : product_codelist_id + # } + # } + # } + # } + # return metadata + + + + # DIRTY PRODUCT FIX + # def update_maghg_4326_storage_product(folder, workspace): + # # there is an issue with the product code of the 4326 layers (they need to be update with the 3857 relative one) + # folders = glob.glob(os.path.join(folder, "*")) + # for folder in folders: + # if os.path.isdir(folder): + # files = glob.glob(os.path.join(folder, "*.geotiff")) + # for f in files: + # uid = workspace + ":" + # uid += "_".join(get_filename(f).lower().split("_")[:-1]) + "_3857" + # print uid + # try: + # metadata = data_manager.metadata_manager.get_by_uid(uid) + # print metadata["dsd"]["contextExtension"]["distribution"][0]["uid"] + # except Exception, e: + # print "ERROR: ", e + + #data_manager.metadata_manager.update_metadata() + + + +#harvest_maghg_4326() +#harvest_maghg_3857() +#harvest_maghg_3857_updated_defaultstyle() + +# maghg_4326_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/storage" +# update_maghg_4326_storage_product(maghg_4326_folder, workspace) diff --git a/geobricks_data_scripts/utils/harvest/harvest_rasters.py b/geobricks_data_scripts/utils/harvest/harvest_rasters.py index 9d2d431..82ab755 100644 --- a/geobricks_data_scripts/utils/harvest/harvest_rasters.py +++ b/geobricks_data_scripts/utils/harvest/harvest_rasters.py @@ -123,6 +123,6 @@ def parse_filename(filename, map_projection_code, product_code=None): # TODO: check prj def check_prj(prj): - print "Check if prj is valid or is the name" + #print "Check if prj is valid" return True diff --git a/geobricks_data_scripts/utils/harvest/publish_harvest.py b/geobricks_data_scripts/utils/harvest/publish_harvest.py index cad1405..6082d88 100644 --- a/geobricks_data_scripts/utils/harvest/publish_harvest.py +++ b/geobricks_data_scripts/utils/harvest/publish_harvest.py @@ -1,5 +1,6 @@ from geobricks_common.core.log import logger from geobricks_data_scripts.utils.harvest.harvest_rasters import harvest_raster_folder +import sys log = logger(__file__) @@ -7,7 +8,6 @@ # TODO: apply publish on storage and geoserver def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, publish_on_storage=True, update_links=True, metadata_json=None): metadatas = harvest_raster_folder(folder) - print metadatas # harvest 3857 for publication for metadata in metadatas: @@ -20,7 +20,6 @@ def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, p metadata = publish_geoserver(data_manager, metadata["path"], metadata) log.info(metadata) - # harvest 4326 (or others) to storage # it's done after the 3857 it's sure the metadata will be there for metadata in metadatas: diff --git a/geobricks_data_scripts/utils/metadata/metadata.py b/geobricks_data_scripts/utils/metadata/metadata.py index 7f89996..0fd4a84 100644 --- a/geobricks_data_scripts/utils/metadata/metadata.py +++ b/geobricks_data_scripts/utils/metadata/metadata.py @@ -8,6 +8,7 @@ def create_metadata(metadata): log.info(metadata) + map_projection_code = str(metadata["map_projection_code"]) product_codelist_id = "layers_products" if "product_codelist_id" not in metadata else metadata["layers_products"] product_codelist_version = "1.0" if "product_codelist_version" not in metadata else metadata["product_codelist_version"] @@ -27,7 +28,7 @@ def create_metadata(metadata): "projection": { "idCodeList": "mapProjections", "version": "1.0", - "codes": [{"code": metadata["map_projection_code"]}] + "codes": [{"code": map_projection_code}] } } }, From 07540e8b2b1ca1ffc89c5ede255ba5150cbeead9 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Mon, 30 Mar 2015 16:22:24 +0200 Subject: [PATCH 21/39] updated ghg import --- .../dev/data/ghg/delete/__init__.py | 1 + .../delete/data_manager_delete_all_example.py | 14 ++ .../dev/data/ghg/delete/delete_a_layer.py | 11 + .../delete/data_manager_delete_all_example.py | 5 +- .../dev/data/ghg/publish/publish_data.py | 238 +++++++++--------- .../utils/harvest/publish_harvest.py | 59 +++-- 6 files changed, 181 insertions(+), 147 deletions(-) create mode 100644 geobricks_data_scripts/dev/data/ghg/delete/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py create mode 100644 geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py diff --git a/geobricks_data_scripts/dev/data/ghg/delete/__init__.py b/geobricks_data_scripts/dev/data/ghg/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py new file mode 100644 index 0000000..15e7983 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py @@ -0,0 +1,14 @@ + +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +for layer in layers: + try: + print layers + if "ghg" in layer["dsd"]["workspace"]: + data_manager.delete(layer["uid"]) + #data_manager.delete(layer["uid"]) + except Exception, e: + print e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py b/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py new file mode 100644 index 0000000..f9c5967 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/delete/delete_a_layer.py @@ -0,0 +1,11 @@ + +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() + +# data_manager.delete("ghg:dm_burning_savanna_2002_3857") +# data_manager.delete("ghg:ch4_emissions_burning_otherforests_2004_3857") + +data_manager.delete("test:jrc_climate_zone_025deg_3857") +# data_manager.delete("test:cultivation_organic_soils_-_croplands_3857") +# data_manager.delete("cultivation_organic_soils_-_croplands_4326") \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/old/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/dev/data/ghg/old/delete/data_manager_delete_all_example.py index 934a485..47722dc 100644 --- a/geobricks_data_scripts/dev/data/ghg/old/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/dev/data/ghg/old/delete/data_manager_delete_all_example.py @@ -5,8 +5,9 @@ layers = data_manager.get_all_layers() for layer in layers: try: - print layer["uid"] - data_manager.delete(layer["uid"]) + if "ghg" in layer["dsd"]["workspace"]: + data_manager.delete(layer["uid"]) + #data_manager.delete(layer["uid"]) except Exception, e: print e pass \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py b/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py index 45a2ca3..c922c48 100644 --- a/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py +++ b/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py @@ -1,128 +1,126 @@ -import glob import os +import glob +from geobricks_common.core.log import logger from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager -from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style +log = logger(__file__) data_manager = get_data_manager() -lang = "EN" workspace = "ghg" -def get_maghg_defaultstyle_name(filename): - sldname = "" - if "FUELBIOMASS" in filename.upper(): - sldname = "fuelbiomass" - elif "CH4_EF" in filename.upper(): - sldname = "ch4_ef" - elif "CO2_EF" in filename.upper(): - sldname = "co2_ef" - elif "N2O_EF" in filename.upper(): - sldname = "n2o_ef" - elif "CH4" in filename.upper(): - sldname = "ch4" - elif "CO2" in filename.upper(): - sldname = "co2" - elif "DM" in filename.upper(): - sldname = "dm" - elif "N2O" in filename.upper(): - sldname = "n2o" - return sldname + "_emissions_burning_" + lang - -# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/Cultivation_organic_soils_croplands/Cultivation_organic_soils_-_croplands/", workspace) -# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GriddedLivestock/gridded_livestock_of_the_world_v_201/", workspace) -# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/JRC_CLIMATE_ZONE/JRC_climate_zone/", workspace) -# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GLC2000/glc2000_v1/", workspace) -# harvest_folder(data_manager, "/home/vortex/Deskt/op/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/GEZ/global_ecological_zones_GEZ/", workspace) - - -def harvest_burned_areas(src_folder, output_folder="output", ): - print "TODO:" - -# GHG-MAG -# 4326 -def harvest_maghg_4326(): - maghg_4326_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/storage" - folders = glob.glob(os.path.join(maghg_4326_folder, "*")) - for folder in folders: - if os.path.isdir(folder): - harvest_folder(data_manager, folder, workspace) - -# 3857 -def harvest_maghg_3857(): - maghg_3857_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/geoserver" - folders = glob.glob(os.path.join(maghg_3857_folder, "*")) - for folder in folders: - if os.path.isdir(folder): - harvest_folder(data_manager, folder, workspace) - - -def harvest_maghg_3857_updated_defaultstyle(): - maghg_3857_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/geoserver" - folders = glob.glob(os.path.join(maghg_3857_folder, "*")) - all_metadatas = [] - for folder in folders: - if os.path.isdir(folder): - all_metadatas.append(harvest_folder(data_manager, folder, workspace, False, False, False)) - - # update the metadata retrieved - for metadatas in all_metadatas: - for metadata in metadatas: - try: - print metadata["dsd"]["layerName"] - default_style = get_maghg_defaultstyle_name(metadata["dsd"]["layerName"]) - data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], default_style) - except Exception, e: - print "ERRRROOOOO!!!!!" - print e - - - # def create_storage_json_metadata(src_folder, code, product_codelist_id="layers_products"): - # for glob=glob - # - # - # def create_product_json(code, product_codelist_id="layers_products"): - # metadata = { - # "meContent" : { - # "seCoverage" : { - # "coverageSectors" : { - # "version" : "1.0", - # "codes" : [ { - # "code" : code - # } ], - # "idCodeList" : product_codelist_id - # } - # } - # } - # } - # return metadata - - - - # DIRTY PRODUCT FIX - # def update_maghg_4326_storage_product(folder, workspace): - # # there is an issue with the product code of the 4326 layers (they need to be update with the 3857 relative one) - # folders = glob.glob(os.path.join(folder, "*")) - # for folder in folders: - # if os.path.isdir(folder): - # files = glob.glob(os.path.join(folder, "*.geotiff")) - # for f in files: - # uid = workspace + ":" - # uid += "_".join(get_filename(f).lower().split("_")[:-1]) + "_3857" - # print uid - # try: - # metadata = data_manager.metadata_manager.get_by_uid(uid) - # print metadata["dsd"]["contextExtension"]["distribution"][0]["uid"] - # except Exception, e: - # print "ERROR: ", e - - #data_manager.metadata_manager.update_metadata() - - - -#harvest_maghg_4326() -#harvest_maghg_3857() -#harvest_maghg_3857_updated_defaultstyle() - -# maghg_4326_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/MAGHG-data/OUTPUT/storage" -# update_maghg_4326_storage_product(maghg_4326_folder, workspace) +def default_import(src_folder, defaultStyle=None): + metadata_json = None + if defaultStyle is not None: + metadata_json = { + "dsd": { + "defaultStyle": defaultStyle + } + } + harvest_folder(data_manager, src_folder, workspace, True, True, True, metadata_json) + + +def import_raster_3857(): + print "Import Raster 3857--------------------" + base_folder = "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/3857_display_only/" + + # # CH4 Emissions Burning + # default_import(base_folder + "CH4_Emissions_Burning_ClosedShrublands", "ch4_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CH4_Emissions_Burning_Grasslands", "ch4_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CH4_Emissions_Burning_HumidTropicalForests", "ch4_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CH4_Emissions_Burning_OtherForests", "ch4_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CH4_Emissions_Burning_Peatlands", "ch4_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CH4_Emissions_Burning_Savanna", "ch4_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CH4_Emissions_Burning_WoodySavanna", "ch4_gfed4ba_emissions_burning_EN") + # + # # CO2 Emissions Burning + # default_import(base_folder + "CO2_Emissions_Burning_ClosedShrublands", "co2_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CO2_Emissions_Burning_Grasslands", "co2_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CO2_Emissions_Burning_HumidTropicalForests", "co2_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CO2_Emissions_Burning_OtherForests", "co2_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CO2_Emissions_Burning_Peatlands", "co2_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CO2_Emissions_Burning_Savanna", "co2_gfed4ba_emissions_burning_EN") + # default_import(base_folder + "CO2_Emissions_Burning_WoodySavanna", "co2_gfed4ba_emissions_burning_EN") + # + # # CO2_CultivationOrganicSoils (The layers have a json file for the style) + # default_import(base_folder + "CO2_CultivationOrganicSoils") + # + # # CultivationOrganicSoils (The layers have a json file for the style) + # default_import(base_folder + "CultivationOrganicSoils") + # + # # DM + # default_import(base_folder + "DM_Burning_ClosedShrublands", "dm_gfed4_burning_EN") + # default_import(base_folder + "DM_Burning_Grasslands", "dm_gfed4_burning_EN") + # default_import(base_folder + "DM_Burning_HumidTropicalForests", "dm_gfed4_burning_EN") + # default_import(base_folder + "DM_Burning_OpenShrublands", "dm_gfed4_burning_EN") + # default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") + # default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") + # default_import(base_folder + "DM_Burning_Peatlands", "dm_gfed4_burning_EN") + # default_import(base_folder + "DM_Burning_Savanna", "dm_gfed4_burning_EN") + # default_import(base_folder + "DM_Burning_WoodySavanna", "dm_gfed4_burning_EN") + # + # # ExtraTropical_Forests_EF (The layers have a json file for the style) + # default_import(base_folder + "ExtraTropical_Forests_EF") + # + # # FuelBiomass (The layers have a json file for the style) + # default_import(base_folder + "FuelBiomass") + # + # # GEZ2010 + # default_import(base_folder + "GEZ2010") + # + # # GFED4 Burned Areas + # default_import(base_folder + "GFED4_BurnedAreas_ClosedShrubland", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_DeciduousBroadleafForest", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_DeciduousNeedleleafForest", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_Deforestation", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_EvergreenBroadleafForest", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_EvergreenNeedleleafForest", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_Grassland", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_MixedForest", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_OpenShrubland", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_OtherForest", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_Peatlands", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_Savanna", "gfed4_burnedarea_EN") + # default_import(base_folder + "GFED4_BurnedAreas_WoodySavanna", "gfed4_burnedarea_EN") + # + # # GLC2000 + # default_import(base_folder + "GLC2000") + # + # # GriddedLivestock + # default_import(base_folder + "GriddedLivestock", "griddedlivestock_heads_EN") + # + # # HWSD + # default_import(base_folder + "HWSD", "hwsd_histosols_area_EN") + # + # # JRC_Climate + # default_import(base_folder + "JRC_Climate", "jrc_climate_zones_EN") + # + # N2O + # N.B. "N2O_CultivationOrganicSoils" ha come stile "cultivation_organicsoils_n2o_emissions" + default_import(base_folder + "N2O_CultivationOrganicSoils", "cultivation_organicsoils_n2o_emissions_EN") + default_import(base_folder + "N2O_Emissions_Burning_ClosedShrublands", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_Grasslands", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_HumidTropicalForests", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_OpenShrublands", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_OtherForests", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_Peatlands", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_Savanna", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_WoodySavanna", "n2o_gfed4ba_emissions_burning_EN") + + # Burning Peatlands (The layers have a json file for the style) + default_import(base_folder + "Peatlands_EF") + + # Savanna+Grassland_EF (The layers have a json file for the style) + default_import(base_folder + "Savanna_and_Grassland_EF") + + # Emission Factor (The layers have a json file for the style) + default_import(base_folder + "Tropical_Forests_EF") + + +if __name__ == '__main__': + + import_raster_3857() + + + diff --git a/geobricks_data_scripts/utils/harvest/publish_harvest.py b/geobricks_data_scripts/utils/harvest/publish_harvest.py index 6082d88..a8c4d60 100644 --- a/geobricks_data_scripts/utils/harvest/publish_harvest.py +++ b/geobricks_data_scripts/utils/harvest/publish_harvest.py @@ -1,4 +1,5 @@ from geobricks_common.core.log import logger +from geobricks_common.core.utils import dict_merge from geobricks_data_scripts.utils.harvest.harvest_rasters import harvest_raster_folder import sys @@ -11,7 +12,15 @@ def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, p # harvest 3857 for publication for metadata in metadatas: + log.info(metadata) + + if metadata_json is not None: + metadata = dict_merge(metadata, metadata_json) + + print metadata + log.info(metadata) + # check epsg if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": if "workspace" not in metadata["dsd"]: @@ -22,31 +31,31 @@ def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, p # harvest 4326 (or others) to storage # it's done after the 3857 it's sure the metadata will be there - for metadata in metadatas: - log.info(metadata) - if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": - if publish_on_storage: - metadata = publish_storage(data_manager, metadata["path"], metadata, workspace) - log.info(metadata) - - log.info(metadatas) - if update_links: - # link 3857 layer to 4326 distribution layer - for metadata in metadatas: - log.info(metadata) - # check epsg - if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": - # TODO: get code - update_dsd_layer_to_distribution_layer(data_manager, metadata) - - - # link 4326 (or other Proj EPSG) layer to 3857 visualization layer - for metadata in metadatas: - log.info(metadata) - # check epsg - if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": - # TODO: get code - update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace) + # for metadata in metadatas: + # log.info(metadata) + # if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": + # if publish_on_storage: + # metadata = publish_storage(data_manager, metadata["path"], metadata, workspace) + # log.info(metadata) + + # log.info(metadatas) + # if update_links: + # # link 3857 layer to 4326 distribution layer + # for metadata in metadatas: + # log.info(metadata) + # # check epsg + # if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": + # # TODO: get code + # update_dsd_layer_to_distribution_layer(data_manager, metadata) + # + # + # # link 4326 (or other Proj EPSG) layer to 3857 visualization layer + # for metadata in metadatas: + # log.info(metadata) + # # check epsg + # if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": + # # TODO: get code + # update_dsd_layer_to_visualization_layer(data_manager, metadata, workspace) return metadatas From df2e69be43e8a13a6951aa842bff519c44b5ab63 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Mon, 13 Apr 2015 18:36:45 +0200 Subject: [PATCH 22/39] extraction test --- .../dev/data/ghg/geoprocess/__init__.py | 1 + .../dev/data/ghg/geoprocess/extract_gaul.py | 1 + .../ghg/geoprocess/extract_layers_by_gaul.py | 1 + .../geoprocess/extract_layers_with_shapely.py | 31 +++++++++++++++++++ .../extract_layers_with_shapely_new.py | 31 +++++++++++++++++++ .../dev/data/ghg/geoprocess/fast_index.py | 1 + .../dev/data/ghg/process_data/__init__.py | 1 + .../dev/data/ghg/process_data/process_data.py | 1 + .../dev/data/ghg/search/__init__.py | 1 + .../dev/data/ghg/search/search_layer.py | 1 + 10 files changed, 70 insertions(+) create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/extract_gaul.py create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_by_gaul.py create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely.py create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely_new.py create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py create mode 100644 geobricks_data_scripts/dev/data/ghg/process_data/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/process_data/process_data.py create mode 100644 geobricks_data_scripts/dev/data/ghg/search/__init__.py create mode 100644 geobricks_data_scripts/dev/data/ghg/search/search_layer.py diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/__init__.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_gaul.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_gaul.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_gaul.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_by_gaul.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_by_gaul.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_by_gaul.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely.py new file mode 100644 index 0000000..0aa7d96 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely.py @@ -0,0 +1,31 @@ + + +# working +import fiona +from shapely.geometry import shape + +input_gaul = '/home/vortex/Desktop/LAYERS/GAUL/simone/gaul0_2008.shp' +input_shp = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1997_3857.shp' +filapath = '/home/vortex/Desktop/LAYERS/GAUL/simone/ita.shp' + +with fiona.open(input_gaul) as gaul: + with fiona.open(input_shp) as shp: + sink_schema = shp.schema.copy() + with fiona.open( + filapath, 'w', + crs=shp.crs, + driver=shp.driver, + schema=sink_schema, + ) as sink: + for g in gaul: + if g['properties'].get('ADM0_NAME') == 'Italy': + print g['properties'].get('ADM0_CODE'), g['properties'].get('ADM0_NAME') + geom_gaul = shape(g['geometry']) + for s in shp: + #print s['properties'] + geom_shp = shape(s['geometry']) + if geom_gaul.intersection(geom_shp): + sink.write(s) + print "daje" + break + diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely_new.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely_new.py new file mode 100644 index 0000000..0aa7d96 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely_new.py @@ -0,0 +1,31 @@ + + +# working +import fiona +from shapely.geometry import shape + +input_gaul = '/home/vortex/Desktop/LAYERS/GAUL/simone/gaul0_2008.shp' +input_shp = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1997_3857.shp' +filapath = '/home/vortex/Desktop/LAYERS/GAUL/simone/ita.shp' + +with fiona.open(input_gaul) as gaul: + with fiona.open(input_shp) as shp: + sink_schema = shp.schema.copy() + with fiona.open( + filapath, 'w', + crs=shp.crs, + driver=shp.driver, + schema=sink_schema, + ) as sink: + for g in gaul: + if g['properties'].get('ADM0_NAME') == 'Italy': + print g['properties'].get('ADM0_CODE'), g['properties'].get('ADM0_NAME') + geom_gaul = shape(g['geometry']) + for s in shp: + #print s['properties'] + geom_shp = shape(s['geometry']) + if geom_gaul.intersection(geom_shp): + sink.write(s) + print "daje" + break + diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/process_data/__init__.py b/geobricks_data_scripts/dev/data/ghg/process_data/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/process_data/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/process_data/process_data.py b/geobricks_data_scripts/dev/data/ghg/process_data/process_data.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/process_data/process_data.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/search/__init__.py b/geobricks_data_scripts/dev/data/ghg/search/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/search/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/search/search_layer.py b/geobricks_data_scripts/dev/data/ghg/search/search_layer.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/search/search_layer.py @@ -0,0 +1 @@ +__author__ = 'vortex' From 17dc24407cb004a2ca260266879db3d86085b5d3 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Mon, 13 Apr 2015 18:36:56 +0200 Subject: [PATCH 23/39] extraction test --- .../dev/data/ghg/geoprocess/extract_gaul.py | 43 +++++++++++- .../ghg/geoprocess/extract_layers_by_gaul.py | 68 ++++++++++++++++++- .../extract_layers_with_shapely_new.py | 39 ++++++----- .../dev/data/ghg/geoprocess/fast_index.py | 40 ++++++++++- 4 files changed, 171 insertions(+), 19 deletions(-) diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_gaul.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_gaul.py index 22d574d..c713d29 100644 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_gaul.py +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_gaul.py @@ -1 +1,42 @@ -__author__ = 'vortex' +import glob +import fiona +import pprint +import os +import shutil +from geobricks_common.core.filesystem import get_filename + +# extract all gaul countries +def extract_gaul_shp(input_path, output_path): + output_folder = os.path.join(output_path) + if os.path.isdir(output_folder): + shutil.rmtree(output_folder) + os.makedirs(output_folder) + + with fiona.open(input_path) as source: + print source.crs + sink_schema = source.schema.copy() + + for f in source: + #print f['properties'] + # print f['properties'].get('ADM0_CODE') + # print f['properties'].get('ADM0_NAME') + if f['properties'].get('DISP_AREA') is not 'NO': + print f['properties'].get('ADM0_CODE'), f['properties'].get('ADM0_NAME') + filapath = os.path.join(output_path, str(int(f['properties'].get('ADM0_CODE'))) + ".shp") + print filapath + with fiona.open( + filapath, 'w', + crs=source.crs, + driver=source.driver, + schema=sink_schema, + ) as sink: + sink.write(f) + + #print s + #print len(list(shp)) + +#extract_gaul_shp('/home/vortex/Desktop/LAYERS/GAUL/GAUL_2014/4326/g2014_2013_0/G2014_2013_0.shp', '/home/vortex/Desktop/LAYERS/GAUL/GAUL_2014/4326/g2014_2013_0/countries') + +input_gaul_folder = '/home/vortex/Desktop/LAYERS/GAUL/GAUL0_FAOSTAT_3857_PROSPERI/GAUL_FAOSTAT_3857_PROSPERI.shp' +output_gaul_folder = '/home/vortex/Desktop/LAYERS/GAUL/GAUL0_FAOSTAT_3857_PROSPERI/countries/' +extract_gaul_shp(input_gaul_folder, output_gaul_folder) \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_by_gaul.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_by_gaul.py index 22d574d..3757a5a 100644 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_by_gaul.py +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_by_gaul.py @@ -1 +1,67 @@ -__author__ = 'vortex' +import os +import glob +import shutil +from geobricks_gis_vector.core.vector import crop_shp_by_shp +from geobricks_common.core.filesystem import get_filename +import time + + +def extract_layer_by_all_countries(input_folder, gaul_folder, base_output_folder): + start = time.time() + shp_files = glob.glob(os.path.join(input_folder, "*.shp")) + print "Shp input files: ", str(len(shp_files)) + gaul_shp_files = glob.glob(os.path.join(gaul_folder, "*.shp")) + print "Shp gaul files: ", str(len(gaul_shp_files)) + print "Shp output files: ", str(len(shp_files) * len(gaul_shp_files)) + counter = 0 + for shp_file in shp_files: + for gaul_shp_file in gaul_shp_files: + print "Clipping: " + get_filename(shp_file), get_filename(gaul_shp_file) + output_folder = os.path.join(base_output_folder, get_filename(gaul_shp_file)) + if not os.path.exists(output_folder): + os.makedirs(output_folder) + counter = counter +1 + extract_layer(shp_file, gaul_shp_file, output_folder) + print counter + end = time.time() + print "All processing done in: ", str(end - start), " sec" + + +def extract_layer(input_path, crop_shp_path, output_folder): + start = time.time() + + #crop_shp_path = '/home/vortex/Desktop/LAYERS/GAUL/GAUL_2014/3857_WebMercatorAuxiliarySphere/g2014_2013_0/countries/1.shp' + #input_path = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1997_3857.shp' + output_name = get_filename(input_path).lower() + + try: + # Clip file + clipped_output_path = crop_shp_by_shp(input_path, crop_shp_path, output_name) + clipped_folder, clipped_filename_ext, clipped_filename = get_filename(clipped_output_path, True) + + output_files = glob.glob(os.path.join(clipped_folder, "*")) + + # move files to output_folder + for f in output_files: + # copy with overwrite + shutil.copy(f, output_folder) + + # remove clipped files + shutil.rmtree(clipped_folder) + end = time.time() + print "Done in: " + str(end - start), " sec" + except Exception,e: + print "Error: ", str(e) + + + + +# input_folder = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/' +# gaul_folder = '/home/vortex/Desktop/LAYERS/GAUL/GAUL_2014/3857_WebMercatorAuxiliarySphere/g2014_2013_0/countries/' +# output_folder = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/gaul0/' +# +# shutil.rmtree(output_folder) +# extract_layer_by_all_countries(input_folder, gaul_folder, output_folder) + + +extract_layer('/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1998_3857.shp', '/home/vortex/Desktop/LAYERS/GAUL/simone/brazil/brazil.shp', '/home/vortex/Desktop/LAYERS/GAUL/simone/brazil/countries/') \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely_new.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely_new.py index 0aa7d96..0789a9f 100644 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely_new.py +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_with_shapely_new.py @@ -9,23 +9,30 @@ filapath = '/home/vortex/Desktop/LAYERS/GAUL/simone/ita.shp' with fiona.open(input_gaul) as gaul: + # geom_gaul = [] + # for g in gaul: + # geom_gaul.append(shape(g['geometry'])) + # print 'finished geometry' + with fiona.open(input_shp) as shp: sink_schema = shp.schema.copy() with fiona.open( - filapath, 'w', - crs=shp.crs, - driver=shp.driver, - schema=sink_schema, - ) as sink: - for g in gaul: - if g['properties'].get('ADM0_NAME') == 'Italy': - print g['properties'].get('ADM0_CODE'), g['properties'].get('ADM0_NAME') - geom_gaul = shape(g['geometry']) - for s in shp: - #print s['properties'] - geom_shp = shape(s['geometry']) - if geom_gaul.intersection(geom_shp): - sink.write(s) - print "daje" - break + filapath, 'w', + crs=shp.crs, + driver=shp.driver, + schema=sink_schema, + ) as sink: + print 'here' + for s in shp: + geom_shp = shape(s['geometry']) + + print "end" + # for g in geom_gaul: + # if geom_shp.intersection(g): + # sink.write(s) + # print "daje" + + + + diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py index 22d574d..b661e48 100644 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py @@ -1 +1,39 @@ -__author__ = 'vortex' +import shapefile +import shapely + +#Load the shapefile of polygons and convert it to shapely polygon objects +polygons_sf = shapefile.Reader("C:/PolygonShapeFile.shp") +polygon_shapes = polygons_sf.shapes() +polygon_points = [q.points for q in polygon_shapes ] +from shapely.geometry import Polygon +polygons = [Polygon(q) for q in polygon_points] + +#Load the shapefile of points and convert it to shapely point objects +points_sf = shapefile.Reader("C:/PointShapeFile.shp") +point_shapes = points_sf.shapes() +from shapely.geometry import Point +point_coords= [q.points[0] for q in point_shapes ] +points = [Point(q.points[0]) for q in point_shapes ] + +#Build a spatial index based on the bounding boxes of the polygons +from rtree import index +idx = index.Index() + +count = -1 +for q in polygon_shapes: + count +=1 + idx.insert(count, q.bbox) + +#Assign one or more matching polygons to each point +matches = [] +for i in range(len(points)): #Iterate through each point + temp= None + print "Point ", i + #Iterate only through the bounding boxes which contain the point + for j in idx.intersection( point_coords[i]): + #Verify that point is within the polygon itself not just the bounding box + if points[i].within(polygons[j]): + print "Match found! ",j + temp=j + break + matches.append(temp) #Either the first match found, or None for no matches \ No newline at end of file From 06c56eedbb464442dbd3a99854f9c062709bf53a Mon Sep 17 00:00:00 2001 From: simonefenix Date: Thu, 30 Apr 2015 12:33:35 +0200 Subject: [PATCH 24/39] added initial method to zip GHG products --- .../extract_layer_from_intersection.py | 1 + .../geoprocess/extract_layers_fast_index.py | 33 ++++++ ...tract_layers_fast_index_multiprocessing.py | 109 ++++++++++++++++++ .../dev/data/ghg/geoprocess/index.py | 1 + .../prod/data/ghg/createZipProducts.py | 1 + 5 files changed, 145 insertions(+) create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layer_from_intersection.py create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index.py create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index_multiprocessing.py create mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/index.py create mode 100644 geobricks_data_scripts/prod/data/ghg/createZipProducts.py diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layer_from_intersection.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layer_from_intersection.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layer_from_intersection.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index.py new file mode 100644 index 0000000..b4fa876 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index.py @@ -0,0 +1,33 @@ +import fiona +from shapely.geometry import shape +import rtree + +def create_gaul_layers(filepath1, filepath2): + with fiona.open(filepath1, 'r') as layer1: + with fiona.open(filepath2, 'r') as layer2: + print "Preparing GAUL0 Indexes" + features = {} + index = rtree.index.Index() + for feat1 in layer1: + fid = int(feat1['id']) + gaul_code = feat1['properties'].get('ADM0_CODE') + geom1 = shape(feat1['geometry']) + index.insert(fid, geom1.bounds) + features[fid] = (feat1, geom1, gaul_code) + + print "Start searching features" + for feat2 in layer2: + geom2 = shape(feat2['geometry']) + print "-------" + for fid in list(index.intersection(geom2.bounds)): + if fid != int(feat2['id']): + # check geometry intersection + feat1, geom1, gaul_code = features[fid] + print gaul_code + print "End" + +filepath1 = '/home/vortex/Desktop/LAYERS/GAUL/simone/gaul0_2008.shp' +filepath2 = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1998_3857.shp' + + +create_gaul_layers(filepath1, filepath2) diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index_multiprocessing.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index_multiprocessing.py new file mode 100644 index 0000000..9168d74 --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index_multiprocessing.py @@ -0,0 +1,109 @@ +import time +import os +import shutil +from shapely.geometry import shape +import fiona +from fiona import collection +from rtree import index +from geobricks_common.core.filesystem import get_filename +from multiprocessing import Process, Queue, Pool + + + +def build_index(filepath, features, property=None): + with collection(filepath, "r") as shapes: + for s in shapes: + geom = shape(s['geometry']) + code = int(s['properties'].get(property)) + id = int(s['id']) + features[id] = (s, geom, code) + yield (id, geom.bounds, s) + +def build_index_slow(filepath, property): + with fiona.open(filepath, 'r') as layer: + features = {} + idx = index.Index() + for s in layer: + if s['properties'].get('DISP_AREA') == 'NO' and s['properties'].get('STATUS') == 'Member State': + id = int(s['id']) + code = int(s['properties'].get(property)) + geom = shape(s['geometry']) + idx.insert(id, geom.bounds) + features[id] = (s, geom, code) + return idx, features + + +def test(input_shp, outputfolder, idx, features): + + print "Reading input shp: ", input_shp + with fiona.open(input_shp) as shp: + sink_schema = shp.schema.copy() + filespath = {} + #print features + # for f in features: + # f, geom, code = features[f] + # print code + + + cached_outputfiles = {} + # create folder structure + print "Preparing shapefiles" + for f in features: + f, geom, code = features[f] + output_folder_code = os.path.join(outputfolder, str(code)) + # TODO: remove it from here + if os.path.isdir(output_folder_code): + shutil.rmtree(output_folder_code) + os.makedirs(output_folder_code) + + # TODO: write directly? How to track unsed shp? + output_filepath = os.path.join(output_folder_code, get_filename(input_shp) + ".shp") + cached_outputfiles[code] = fiona.open(output_filepath, 'w', crs=shp.crs, driver=shp.driver, schema=sink_schema) + + print 'End Sink files preparation' + + q = Queue() + pool = Pool(2) #use all available cores, otherwise specify the number you want as an argument + processes = [] + print 'Filling shapefiles' + for feat in shp: + geom = shape(feat['geometry']) + for id in list(idx.intersection(geom.bounds)): + if id != int(feat['id']): + # check geometry intersection + feat1, geom1, code = features[id] + + #print feat['id'], str(id), str(code) + + # additional check if the geom is in the geometry (and not only in the bounding box) + if geom1.intersection(geom): + print feat['id'], str(id), str(code) + + # write sink + cached_outputfiles[code].write(feat) + + for f in cached_outputfiles: + cached_outputfiles[f].close() + + + + +input_gaul = '/home/vortex/Desktop/LAYERS/GAUL/GAUL_2014/3857_WebMercatorAuxiliarySphere/GAUL0/G2014_2013_0_mid.shp' +#input_gaul = '/home/vortex/Desktop/LAYERS/GAUL/simone/ita.shp' +input_shp = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1997_3857.shp' +filapath = '/home/vortex/Desktop/LAYERS/GAUL/simone/ita.shp' +output_folder = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/gaul0/' + +print "Preparing index" +start = time.time() +#features = {} +#idx = index.Rtree('bulk', build_index(input_gaul, features, 'ADM0_CODE')) +idx, features = build_index_slow(input_gaul, 'ADM0_CODE') +end = time.time() +print "End index preparation in: " + str(end - start) + " sec" + + +start = time.time() +test(input_shp, output_folder, idx, features) +end = time.time() +print "End Creating shapefiles in: " + str(end - start) + " sec" \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/index.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/index.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/index.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/prod/data/ghg/createZipProducts.py b/geobricks_data_scripts/prod/data/ghg/createZipProducts.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/prod/data/ghg/createZipProducts.py @@ -0,0 +1 @@ +__author__ = 'vortex' From 99dd3d5ff7674059332ad6f8f40d3e096213e72e Mon Sep 17 00:00:00 2001 From: simonefenix Date: Thu, 30 Apr 2015 12:34:01 +0200 Subject: [PATCH 25/39] added initial method to zip GHG products --- .../prod/data/ghg/createZipProducts.py | 116 +++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/geobricks_data_scripts/prod/data/ghg/createZipProducts.py b/geobricks_data_scripts/prod/data/ghg/createZipProducts.py index 22d574d..18ca638 100644 --- a/geobricks_data_scripts/prod/data/ghg/createZipProducts.py +++ b/geobricks_data_scripts/prod/data/ghg/createZipProducts.py @@ -1 +1,115 @@ -__author__ = 'vortex' +import glob +import os +import shutil +from geobricks_common.core.filesystem import createZip + + +def createGHGProducts(obj): + inputPath = obj["inputPath"] + outputPath = obj["outputPath"] + archiveType = obj["archiveType"] + codingSystem = obj["codingSystem"] if "codingSystem" in obj else None + zips = obj["zippit"] + + # Get all country codes (to be used in the folder input and output folder construction + # TODO: how to make generic? + codes_folders = glob.glob(os.path.join(inputPath, "*")) + codes = [] + for folder in codes_folders: + codes.append(os.path.split(folder)[1]) + + # create the zip files + for to_zip in zips: + print to_zip["name"] + outputFilename = to_zip["filename"] + outputFolderName = to_zip["folderName"] + + # for each country code + for code in codes: + + # TODO: get all country codes (to be used in the folder input construction) + + # TODO add to the path gaul code i.e. 4326/product/faostat/12/product_name.zip + # Create Product + outputFolderPath = os.path.join(outputPath, outputFolderName) + if not os.path.isdir(outputFolderPath): + os.makedirs(outputFolderPath) + + # Create Coding System + if codingSystem: + outputFolderPath = os.path.join(outputFolderPath, codingSystem) + if not os.path.isdir(outputFolderPath): + os.makedirs(outputFolderPath) + + # Create output folder and subfolder + outputFolderPath = os.path.join(outputFolderPath, code) + if not os.path.isdir(outputFolderPath): + os.makedirs(outputFolderPath) + + outputFilePath = os.path.join(outputFolderPath, outputFilename) + + # delete file if already exists + if os.path.isfile(outputFilePath): + os.remove(outputFilePath) + + folders_to_zip = [] + for folder in to_zip["folders"]: + # TODO add to the path gaul code i.e. inputPath/12/inputFolder + folders_to_zip.append(os.path.join(inputPath, code, folder)) + + # create zip file + createZip(outputFilePath, folders_to_zip, archiveType) + +# Definition of the zip +obj = { + "inputPath": "/home/vortex/Desktop/LAYERS/ghg/process_data/test_input/COUNTRIES/", + "outputPath": "/home/vortex/Desktop/LAYERS/ghg/process_data/output/4326/", + "archiveType": "zip", + "codingSystem": "faostat", + # this is done in this case for each country + "zippit": [ + { + "name": "FAOSTAT Fire CO2 emission estimates", + "folderName": "co2_emission_estimates", + "filename": "co2_emission_estimates", + "metadataFilesPath": [], + "folders": [ + "CO2_GFED4BA_Emissions_Burning_ClosedShrublands", + "CO2_GFED4BA_Emissions_Burning_OpenShrublands", + ] + }, + { + "name": "DM GFED4 burning peatlands", + "folderName": "dm_gfed4_burning_peatlands", + "filename": "dm_gfed4_burning_peatlands", + "metadataFilesPath": [], + "folders": [ + "DM_GFED4_Burning_Peatlands", + ] + }, + { + "name": "N2O GFED4BA Emissions Burning Peatlands", + "folderName": "n2o_gfed4ba_emissions_burning_peatlands", + "filename": "n2o_gfed4ba_emissions_burning_peatlands", + "metadataFilesPath": [], + "folders": [ + "N2O_GFED4BA_Emissions_Burning_Peatlands", + ] + }, + { + "name": "GFED4_BurnedArea_Peatlands", + "folderName": "gfed4_burnedarea_peatlands", + "filename": "gfed4_burnedarea_peatlands", + "metadataFilesPath": [], + "folders": [ + "GFED4_BurnedArea_Peatlands", + ] + } + ] +} + + +# remove thte output folder and create new output zip files +if os.path.isdir(obj["outputPath"]): + shutil.rmtree(obj["outputPath"]) +createGHGProducts(obj) \ No newline at end of file From e2633080b23368e622ee066c5a22b45f69926d21 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Thu, 30 Apr 2015 12:35:59 +0200 Subject: [PATCH 26/39] updated publish --- .../dev/data/ghg/publish/publish_data.py | 160 ++++++++++-------- .../dev/data/ghg/search/search_layer.py | 28 ++- .../utils/harvest/publish_harvest.py | 16 +- 3 files changed, 120 insertions(+), 84 deletions(-) diff --git a/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py b/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py index c922c48..3d77afe 100644 --- a/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py +++ b/geobricks_data_scripts/dev/data/ghg/publish/publish_data.py @@ -1,8 +1,8 @@ -import os import glob +import os from geobricks_common.core.log import logger from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager -from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder log = logger(__file__) @@ -25,77 +25,77 @@ def import_raster_3857(): print "Import Raster 3857--------------------" base_folder = "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/3857_display_only/" - # # CH4 Emissions Burning - # default_import(base_folder + "CH4_Emissions_Burning_ClosedShrublands", "ch4_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CH4_Emissions_Burning_Grasslands", "ch4_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CH4_Emissions_Burning_HumidTropicalForests", "ch4_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CH4_Emissions_Burning_OtherForests", "ch4_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CH4_Emissions_Burning_Peatlands", "ch4_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CH4_Emissions_Burning_Savanna", "ch4_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CH4_Emissions_Burning_WoodySavanna", "ch4_gfed4ba_emissions_burning_EN") - # - # # CO2 Emissions Burning - # default_import(base_folder + "CO2_Emissions_Burning_ClosedShrublands", "co2_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CO2_Emissions_Burning_Grasslands", "co2_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CO2_Emissions_Burning_HumidTropicalForests", "co2_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CO2_Emissions_Burning_OtherForests", "co2_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CO2_Emissions_Burning_Peatlands", "co2_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CO2_Emissions_Burning_Savanna", "co2_gfed4ba_emissions_burning_EN") - # default_import(base_folder + "CO2_Emissions_Burning_WoodySavanna", "co2_gfed4ba_emissions_burning_EN") - # - # # CO2_CultivationOrganicSoils (The layers have a json file for the style) - # default_import(base_folder + "CO2_CultivationOrganicSoils") - # - # # CultivationOrganicSoils (The layers have a json file for the style) - # default_import(base_folder + "CultivationOrganicSoils") - # - # # DM - # default_import(base_folder + "DM_Burning_ClosedShrublands", "dm_gfed4_burning_EN") - # default_import(base_folder + "DM_Burning_Grasslands", "dm_gfed4_burning_EN") - # default_import(base_folder + "DM_Burning_HumidTropicalForests", "dm_gfed4_burning_EN") - # default_import(base_folder + "DM_Burning_OpenShrublands", "dm_gfed4_burning_EN") - # default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") - # default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") - # default_import(base_folder + "DM_Burning_Peatlands", "dm_gfed4_burning_EN") - # default_import(base_folder + "DM_Burning_Savanna", "dm_gfed4_burning_EN") - # default_import(base_folder + "DM_Burning_WoodySavanna", "dm_gfed4_burning_EN") - # - # # ExtraTropical_Forests_EF (The layers have a json file for the style) - # default_import(base_folder + "ExtraTropical_Forests_EF") - # - # # FuelBiomass (The layers have a json file for the style) - # default_import(base_folder + "FuelBiomass") - # - # # GEZ2010 - # default_import(base_folder + "GEZ2010") - # - # # GFED4 Burned Areas - # default_import(base_folder + "GFED4_BurnedAreas_ClosedShrubland", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_DeciduousBroadleafForest", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_DeciduousNeedleleafForest", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_Deforestation", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_EvergreenBroadleafForest", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_EvergreenNeedleleafForest", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_Grassland", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_MixedForest", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_OpenShrubland", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_OtherForest", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_Peatlands", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_Savanna", "gfed4_burnedarea_EN") - # default_import(base_folder + "GFED4_BurnedAreas_WoodySavanna", "gfed4_burnedarea_EN") - # - # # GLC2000 - # default_import(base_folder + "GLC2000") - # - # # GriddedLivestock - # default_import(base_folder + "GriddedLivestock", "griddedlivestock_heads_EN") - # - # # HWSD - # default_import(base_folder + "HWSD", "hwsd_histosols_area_EN") - # - # # JRC_Climate - # default_import(base_folder + "JRC_Climate", "jrc_climate_zones_EN") - # + # CH4 Emissions Burning + default_import(base_folder + "CH4_Emissions_Burning_ClosedShrublands", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_Grasslands", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_HumidTropicalForests", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_OtherForests", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_Peatlands", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_Savanna", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_WoodySavanna", "ch4_gfed4ba_emissions_burning_EN") + + # CO2 Emissions Burning + default_import(base_folder + "CO2_Emissions_Burning_ClosedShrublands", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_Grasslands", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_HumidTropicalForests", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_OtherForests", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_Peatlands", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_Savanna", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_WoodySavanna", "co2_gfed4ba_emissions_burning_EN") + + # CO2_CultivationOrganicSoils (The layers have a json file for the style) + default_import(base_folder + "CO2_CultivationOrganicSoils") + + # CultivationOrganicSoils (The layers have a json file for the style) + default_import(base_folder + "CultivationOrganicSoils") + + # DM + default_import(base_folder + "DM_Burning_ClosedShrublands", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_Grasslands", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_HumidTropicalForests", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_OpenShrublands", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_Peatlands", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_Savanna", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_WoodySavanna", "dm_gfed4_burning_EN") + + # ExtraTropical_Forests_EF (The layers have a json file for the style) + default_import(base_folder + "ExtraTropical_Forests_EF") + + # FuelBiomass (The layers have a json file for the style) + default_import(base_folder + "FuelBiomass") + + # GEZ2010 + default_import(base_folder + "GEZ2010") + + # GFED4 Burned Areas + default_import(base_folder + "GFED4_BurnedAreas_ClosedShrubland", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_DeciduousBroadleafForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_DeciduousNeedleleafForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_Deforestation", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_EvergreenBroadleafForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_EvergreenNeedleleafForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_Grassland", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_MixedForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_OpenShrubland", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_OtherForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_Peatlands", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_Savanna", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_WoodySavanna", "gfed4_burnedarea_EN") + + # GLC2000 + default_import(base_folder + "GLC2000") + + # GriddedLivestock + default_import(base_folder + "GriddedLivestock", "griddedlivestock_heads_EN") + + # HWSD + default_import(base_folder + "HWSD", "hwsd_histosols_area_EN") + + # JRC_Climate + default_import(base_folder + "JRC_Climate", "jrc_climate_zones_EN") + # N2O # N.B. "N2O_CultivationOrganicSoils" ha come stile "cultivation_organicsoils_n2o_emissions" default_import(base_folder + "N2O_CultivationOrganicSoils", "cultivation_organicsoils_n2o_emissions_EN") @@ -118,9 +118,19 @@ def import_raster_3857(): default_import(base_folder + "Tropical_Forests_EF") -if __name__ == '__main__': +def import_raster_4326(): + print "Import Raster 4326--------------------" + base_folder = "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/4326/*" - import_raster_3857() + folders = glob.glob(base_folder) + for folder in folders: + if os.path.isdir(folder): + harvest_folder(data_manager, folder, workspace, False, True, True) +if __name__ == '__main__': + + # import_raster_3857() + + import_raster_4326() diff --git a/geobricks_data_scripts/dev/data/ghg/search/search_layer.py b/geobricks_data_scripts/dev/data/ghg/search/search_layer.py index 22d574d..8b657c6 100644 --- a/geobricks_data_scripts/dev/data/ghg/search/search_layer.py +++ b/geobricks_data_scripts/dev/data/ghg/search/search_layer.py @@ -1 +1,27 @@ -__author__ = 'vortex' +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +import os + +data_manager = get_data_manager() + +#layers = data_manager.metadata_manager.get_by_layername_workspace("gfed4_burnedarea_mixedforest_2002_3857", "ghg") + +layers = data_manager.metadata_manager.get_all_layers() + +count_4326 = 0 +count_3857 = 0 + +for l in layers: + print l["meReferenceSystem"]["seProjection"]["projection"]["codes"] + if "4326"in l["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"]: + count_4326 += 1 + elif "3857"in l["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"]: + count_3857 += 1 + else: + print "NOOO" + +print count_4326 +print count_3857 + + +#print os.path.isfile("/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/4326/CH4_Emissions_Burning_HumidTropicalForests/CH4_GFED4BA_Emissions_Burning_HumidTropicalForests_1997_4326.tif") + diff --git a/geobricks_data_scripts/utils/harvest/publish_harvest.py b/geobricks_data_scripts/utils/harvest/publish_harvest.py index a8c4d60..16f8774 100644 --- a/geobricks_data_scripts/utils/harvest/publish_harvest.py +++ b/geobricks_data_scripts/utils/harvest/publish_harvest.py @@ -29,14 +29,14 @@ def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, p metadata = publish_geoserver(data_manager, metadata["path"], metadata) log.info(metadata) - # harvest 4326 (or others) to storage - # it's done after the 3857 it's sure the metadata will be there - # for metadata in metadatas: - # log.info(metadata) - # if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": - # if publish_on_storage: - # metadata = publish_storage(data_manager, metadata["path"], metadata, workspace) - # log.info(metadata) + # harvest 4326 (or others) to storage + # it's done after the 3857 it's sure the metadata will be there + for metadata in metadatas: + log.info(metadata) + if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": + if publish_on_storage: + metadata = publish_storage(data_manager, metadata["path"], metadata, workspace) + log.info(metadata) # log.info(metadatas) # if update_links: From 3910a4264cec170e355846b03a21af7756ee228f Mon Sep 17 00:00:00 2001 From: simonefenix Date: Thu, 30 Apr 2015 12:36:21 +0200 Subject: [PATCH 27/39] process vector layers --- .../delete/data_manager_delete_all_example.py | 9 +- .../extract_layer_from_intersection.py | 38 ++++- .../geoprocess/extract_layers_fast_index.py | 142 ++++++++++++++---- ...tract_layers_fast_index_multiprocessing.py | 109 -------------- .../dev/data/ghg/geoprocess/fast_index.py | 65 ++++---- .../dev/data/ghg/geoprocess/index.py | 127 +++++++++++++++- .../dev/data/ghg/process_data/process_data.py | 88 ++++++++++- 7 files changed, 396 insertions(+), 182 deletions(-) delete mode 100644 geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index_multiprocessing.py diff --git a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py index 15e7983..e837fed 100644 --- a/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/dev/data/ghg/delete/data_manager_delete_all_example.py @@ -5,10 +5,11 @@ layers = data_manager.get_all_layers() for layer in layers: try: - print layers - if "ghg" in layer["dsd"]["workspace"]: - data_manager.delete(layer["uid"]) + # print layers + if "workspace" in layer["dsd"] is not None and "ghg" in layer["dsd"]["workspace"]: + print layer["uid"] + #data_manager.delete(layer["uid"]) #data_manager.delete(layer["uid"]) except Exception, e: - print e + #print e pass \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layer_from_intersection.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layer_from_intersection.py index 22d574d..81f5a94 100644 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layer_from_intersection.py +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layer_from_intersection.py @@ -1 +1,37 @@ -__author__ = 'vortex' +from osgeo import ogr +from shapely.wkb import loads +from shapely.geometry import * +import fiona + +#first layer, a polygon shapefile +# first = Polygon() +# # open shapefile +# source1 = ogr.Open("/home/vortex/Desktop/LAYERS/GAUL/GAUL0_FAOSTAT_3857_PROSPERI/countries/1.shp") +# layer1 = source1.GetLayer() +# +# print "1" +# # combination of all the geometries of the layer in a single shapely object +# for element in layer1: +# geom = loads(element.GetGeometryRef().ExportToWkb()) +# first = first.union(geom) +# # second layer, a polygon shapefile +# +# print "2" +# two = Polygon() +# source2 = ogr.Open("/home/vortex/Desktop/LAYERS/ghg/process_data/3857/FuelBiomass/Burning_WoodySavanna_FuelBiomass_3857.shp") +# layer2 = source2.GetLayer() +# for element in layer2: +# geom = loads(element.GetGeometryRef().ExportToWkb()) +# two = two.union(geom) +# +# print "3" +# shp = fiona.open("/home/vortex/Desktop/LAYERS/ghg/process_data/3857/FuelBiomass/Burning_WoodySavanna_FuelBiomass_3857.shp") +# sink_schema = shp.schema.copy() +# print sink_schema +# # intersection between the two layers +# output = fiona.open('/home/vortex/Desktop/LAYERS/ghg/process_data/test/test.hsp', 'w', crs=shp.crs, driver=shp.driver, schema=sink_schema) +# output.write(first.intersection(two)) + +shp = fiona.open("/home/vortex/Desktop/LAYERS/ghg/process_data/3857/FuelBiomass/Burning_WoodySavanna_FuelBiomass_3857.shp") +for feat in shp: + print feat \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index.py index b4fa876..eac748a 100644 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index.py +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index.py @@ -1,33 +1,111 @@ +import time +import sys +import os +import shutil +import glob +from shapely.geometry import shape, mapping import fiona -from shapely.geometry import shape -import rtree - -def create_gaul_layers(filepath1, filepath2): - with fiona.open(filepath1, 'r') as layer1: - with fiona.open(filepath2, 'r') as layer2: - print "Preparing GAUL0 Indexes" - features = {} - index = rtree.index.Index() - for feat1 in layer1: - fid = int(feat1['id']) - gaul_code = feat1['properties'].get('ADM0_CODE') - geom1 = shape(feat1['geometry']) - index.insert(fid, geom1.bounds) - features[fid] = (feat1, geom1, gaul_code) - - print "Start searching features" - for feat2 in layer2: - geom2 = shape(feat2['geometry']) - print "-------" - for fid in list(index.intersection(geom2.bounds)): - if fid != int(feat2['id']): - # check geometry intersection - feat1, geom1, gaul_code = features[fid] - print gaul_code - print "End" - -filepath1 = '/home/vortex/Desktop/LAYERS/GAUL/simone/gaul0_2008.shp' -filepath2 = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1998_3857.shp' - - -create_gaul_layers(filepath1, filepath2) +from fiona import collection +from rtree import index +from geobricks_common.core.filesystem import get_filename +from progressbar import Counter, ProgressBar, Timer + + +def build_index(filepath, features, property=None): + with collection(filepath, "r") as shapes: + for s in shapes: + geom = shape(s['geometry']) + code = int(s['properties'].get(property)) + id = int(s['id']) + features[id] = (s, geom, code) + yield (id, geom.bounds, s) + +def build_index_slow(filepath, property): + with fiona.open(filepath, 'r') as layer: + features = {} + idx = index.Index() + + for s in layer: + if s['properties'].get('DISP_AREA') == None: + id = int(s['id']) + code = int(s['properties'].get(property)) + geom = shape(s['geometry']) + idx.insert(id, geom.bounds) + features[id] = (s, geom, code) + return idx, features + + +def extraxt_data(input_shp, outputfolder, idx, features): + + print "Reading input shp: ", input_shp + with fiona.open(input_shp) as shp: + sink_schema = shp.schema.copy() + + # TODO: check which countries are written (and delete the unused folders) + + # create caching folder structure + cached_outputfiles = {} + cached_written_files = {} + print "Preparing shapefiles" + for f in features: + f, geom, code = features[f] + output_folder_code = os.path.join(outputfolder, str(code)) + if not os.path.isdir(output_folder_code): + #shutil.rmtree(output_folder_code) + os.makedirs(output_folder_code) + + # TODO: write directly? How to track unsed shp? + output_filepath = os.path.join(output_folder_code, get_filename(input_shp) + ".shp") + cached_outputfiles[code] = fiona.open(output_filepath, 'w', crs=shp.crs, driver=shp.driver, schema=sink_schema) + cached_written_files[code] = False + + # print 'End Sink files preparation' + # print 'Processing shapefiles' + + # creating progress bar + widgets = ['Processed: ', Counter(), '/', str(len(shp)), ' in ', Timer()] + pbar = ProgressBar(widgets=widgets, maxval=len(shp)) + pbar.start() + count_tot = 0 + for feat in shp: + count_tot += 1 + pbar.update(count_tot) + + geom = shape(feat['geometry']) + for id in list(idx.intersection(geom.bounds)): + if id != int(feat['id']): + # check geometry intersection + feat1, geom_feature, code = features[id] + + # additional check if the geom is in the geometry (and not only in the bounding box) + # and calculate the intersection + intersection = geom_feature.intersection(geom) + if intersection: + # change the feature with the new feature geometry + feat['geometry'] = mapping(intersection) + # write sink + cached_outputfiles[code].write(feat) + cached_written_files[code] = True + + # destroy progress bar + pbar.finish() + + # Close cached outputfiles + for f in cached_outputfiles: + cached_outputfiles[f].close() + + # Removed not filled files + for f in cached_written_files: + if cached_written_files[f] is False: + folder, filename_ext, filename = get_filename(cached_outputfiles[f].path, True) + files_to_remove = glob.glob(os.path.join(folder, filename + ".*")) + for file_to_remove in files_to_remove: + os.remove(file_to_remove) + + # remove folder if empty + if len(os.listdir(folder)) == 0: + #print folder + shutil.rmtree(folder) + + + diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index_multiprocessing.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index_multiprocessing.py deleted file mode 100644 index 9168d74..0000000 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/extract_layers_fast_index_multiprocessing.py +++ /dev/null @@ -1,109 +0,0 @@ -import time -import os -import shutil -from shapely.geometry import shape -import fiona -from fiona import collection -from rtree import index -from geobricks_common.core.filesystem import get_filename -from multiprocessing import Process, Queue, Pool - - - -def build_index(filepath, features, property=None): - with collection(filepath, "r") as shapes: - for s in shapes: - geom = shape(s['geometry']) - code = int(s['properties'].get(property)) - id = int(s['id']) - features[id] = (s, geom, code) - yield (id, geom.bounds, s) - -def build_index_slow(filepath, property): - with fiona.open(filepath, 'r') as layer: - features = {} - idx = index.Index() - for s in layer: - if s['properties'].get('DISP_AREA') == 'NO' and s['properties'].get('STATUS') == 'Member State': - id = int(s['id']) - code = int(s['properties'].get(property)) - geom = shape(s['geometry']) - idx.insert(id, geom.bounds) - features[id] = (s, geom, code) - return idx, features - - -def test(input_shp, outputfolder, idx, features): - - print "Reading input shp: ", input_shp - with fiona.open(input_shp) as shp: - sink_schema = shp.schema.copy() - filespath = {} - #print features - # for f in features: - # f, geom, code = features[f] - # print code - - - cached_outputfiles = {} - # create folder structure - print "Preparing shapefiles" - for f in features: - f, geom, code = features[f] - output_folder_code = os.path.join(outputfolder, str(code)) - # TODO: remove it from here - if os.path.isdir(output_folder_code): - shutil.rmtree(output_folder_code) - os.makedirs(output_folder_code) - - # TODO: write directly? How to track unsed shp? - output_filepath = os.path.join(output_folder_code, get_filename(input_shp) + ".shp") - cached_outputfiles[code] = fiona.open(output_filepath, 'w', crs=shp.crs, driver=shp.driver, schema=sink_schema) - - print 'End Sink files preparation' - - q = Queue() - pool = Pool(2) #use all available cores, otherwise specify the number you want as an argument - processes = [] - print 'Filling shapefiles' - for feat in shp: - geom = shape(feat['geometry']) - for id in list(idx.intersection(geom.bounds)): - if id != int(feat['id']): - # check geometry intersection - feat1, geom1, code = features[id] - - #print feat['id'], str(id), str(code) - - # additional check if the geom is in the geometry (and not only in the bounding box) - if geom1.intersection(geom): - print feat['id'], str(id), str(code) - - # write sink - cached_outputfiles[code].write(feat) - - for f in cached_outputfiles: - cached_outputfiles[f].close() - - - - -input_gaul = '/home/vortex/Desktop/LAYERS/GAUL/GAUL_2014/3857_WebMercatorAuxiliarySphere/GAUL0/G2014_2013_0_mid.shp' -#input_gaul = '/home/vortex/Desktop/LAYERS/GAUL/simone/ita.shp' -input_shp = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1997_3857.shp' -filapath = '/home/vortex/Desktop/LAYERS/GAUL/simone/ita.shp' -output_folder = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/gaul0/' - -print "Preparing index" -start = time.time() -#features = {} -#idx = index.Rtree('bulk', build_index(input_gaul, features, 'ADM0_CODE')) -idx, features = build_index_slow(input_gaul, 'ADM0_CODE') -end = time.time() -print "End index preparation in: " + str(end - start) + " sec" - - -start = time.time() -test(input_shp, output_folder, idx, features) -end = time.time() -print "End Creating shapefiles in: " + str(end - start) + " sec" \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py index b661e48..cede339 100644 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/fast_index.py @@ -1,39 +1,36 @@ -import shapefile -import shapely +import fiona +from shapely.geometry import shape +import rtree -#Load the shapefile of polygons and convert it to shapely polygon objects -polygons_sf = shapefile.Reader("C:/PolygonShapeFile.shp") -polygon_shapes = polygons_sf.shapes() -polygon_points = [q.points for q in polygon_shapes ] -from shapely.geometry import Polygon -polygons = [Polygon(q) for q in polygon_points] +def create_gaul_layers(filepath1, filepath2): + with fiona.open(filepath1, 'r') as layer1: + with fiona.open(filepath2, 'r') as layer2: + print "Preparing GAUL0 Indexes" + features = {} + index = rtree.index.Index() + for feat1 in layer1: + fid = int(feat1['id']) + print + gaul_code = feat1['properties'].get('ADM0_CODE') + geom1 = shape(feat1['geometry']) + index.insert(fid, geom1.bounds) + features[fid] = (feat1, geom1, gaul_code) -#Load the shapefile of points and convert it to shapely point objects -points_sf = shapefile.Reader("C:/PointShapeFile.shp") -point_shapes = points_sf.shapes() -from shapely.geometry import Point -point_coords= [q.points[0] for q in point_shapes ] -points = [Point(q.points[0]) for q in point_shapes ] + print "Start searching features" + for feat2 in layer2: + geom2 = shape(feat2['geometry']) + print "-------" + for fid in list(index.intersection(geom2.bounds)): + if fid != int(feat2['id']): + # check geometry intersection + feat1, geom1, gaul_code = features[fid] + print gaul_code + print "End" -#Build a spatial index based on the bounding boxes of the polygons -from rtree import index -idx = index.Index() -count = -1 -for q in polygon_shapes: - count +=1 - idx.insert(count, q.bbox) -#Assign one or more matching polygons to each point -matches = [] -for i in range(len(points)): #Iterate through each point - temp= None - print "Point ", i - #Iterate only through the bounding boxes which contain the point - for j in idx.intersection( point_coords[i]): - #Verify that point is within the polygon itself not just the bounding box - if points[i].within(polygons[j]): - print "Match found! ",j - temp=j - break - matches.append(temp) #Either the first match found, or None for no matches \ No newline at end of file +filepath1 = '/home/vortex/Desktop/LAYERS/GAUL/simone/gaul0_2008.shp' +filepath2 = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/CH4_Emissions_Burning_ClosedShrublands/CH4_GFED4BA_Emissions_Burning_ClosedShrublands_1998_3857.shp' + + +create_gaul_layers(filepath1, filepath2) diff --git a/geobricks_data_scripts/dev/data/ghg/geoprocess/index.py b/geobricks_data_scripts/dev/data/ghg/geoprocess/index.py index 22d574d..2df489e 100644 --- a/geobricks_data_scripts/dev/data/ghg/geoprocess/index.py +++ b/geobricks_data_scripts/dev/data/ghg/geoprocess/index.py @@ -1 +1,126 @@ -__author__ = 'vortex' +import time +import glob +import os +import shutil +from geobricks_data_scripts.dev.data.ghg.geoprocess.extract_layers_fast_index import build_index_slow, extraxt_data + + +def get_index(path, property): + print "Preparing index" + start = time.time() + idx, features = build_index_slow(path, property) + end = time.time() + print "End index preparation in: " + str(end - start) + " sec" + return idx, features + + + +def process_folder(idx, features, input_folder, output_folder): + files = glob.glob(os.path.join(input_folder, "*.shp")) + + # make the output folder + if os.path.isdir(output_folder): + shutil.rmtree(output_folder) + os.makedirs(output_folder) + + for input_shp in files: + print "Processing: ", input_shp + start = time.time() + extraxt_data(input_shp, output_folder, idx, features) + end = time.time() + print "End processing in: " + str(end - start) + " sec of ", input_shp + print "------------------" + + + +if __name__ == "__main__": + + # TODO: Per paolo + # folder con problemi + # /home/vortex/Desktop/LAYERS/ghg/process_data/3857/Peatlands_EF/ + + + input_gaul = '/home/vortex/Desktop/LAYERS/GAUL/GAUL0_FAOSTAT_3857_PROSPERI/GAUL_FAOSTAT_3857_PROSPERI.shp' + idx, features = get_index(input_gaul, 'ADM0_CODE') + + # base input folder + base_input_folder = '/home/vortex/Desktop/LAYERS/ghg/process_data/3857/' + base_output_folder = '/home/vortex/Desktop/LAYERS/ghg/process_data/test_output/' + + folders = glob.glob(base_input_folder + "*") + for folder in folders: + if os.path.isdir(folder): + dirname = os.path.basename(folder) + input_folder = folder + output_folder = os.path.join(base_output_folder, dirname) + process_folder(idx, features, input_folder, output_folder) + + + # # CultivationOrganicSoils + # path = 'CultivationOrganicSoils' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # DM_Burning_ClosedShrublands + # path = 'DM_Burning_ClosedShrublands' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # DM_Burning_Grasslands + # path = 'DM_Burning_Grasslands' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # fuel biomass + # path = 'FuelBiomass' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # Savanna_and_Grassland_EF + # path = 'Savanna_and_Grassland_EF' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # Tropical_Forests_EF + # path = 'Tropical_Forests_EF' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # Tropical_Forests_EF + # path = 'Tropical_Forests_EF' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # CH4_Emissions_Burning_ClosedShrublands/ + # path = 'CH4_Emissions_Burning_ClosedShrublands' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # CH4_Emissions_Burning_Grasslands + # path = 'CH4_Emissions_Burning_Grasslands' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # CH4_Emissions_Burning_HumidTropicalForests + # path = 'CH4_Emissions_Burning_HumidTropicalForests' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + # + # # GEZ2010 + # path = 'GEZ2010' + # input_folder = os.path.join(base_input_folder, path) + # output_folder = os.path.join(base_output_folder, path) + # process_folder(idx, features, input_folder, output_folder) + + + diff --git a/geobricks_data_scripts/dev/data/ghg/process_data/process_data.py b/geobricks_data_scripts/dev/data/ghg/process_data/process_data.py index 22d574d..8ce3bd5 100644 --- a/geobricks_data_scripts/dev/data/ghg/process_data/process_data.py +++ b/geobricks_data_scripts/dev/data/ghg/process_data/process_data.py @@ -1 +1,87 @@ -__author__ = 'vortex' +import os +import glob +import shutil +from geobricks_common.core.filesystem import get_filename + + + +# Raster Data +def preprocess_raster_data_for_storage(srs_folder, dst_folder, filetype="tif"): + folders = glob.glob(srs_folder +"/*") + for folder in folders: + if os.path.isdir(folder): + files = glob.glob(folder + "/*." + filetype) + for src_file in files: + _move_raster_file_to_storage(dst_folder, src_file) + +# Move the raster to the storage folders +def _move_raster_file_to_storage(dst_folder, src_file, output_filetype="geotiff"): + + src_filename = get_filename(src_file.lower()) + # lowercase (everything is lowercase) + + dst_folder_file = os.path.join(dst_folder, get_filename(src_filename)) + + # create folder + if os.path.isdir(dst_folder_file) is True: + shutil.rmtree(dst_folder_file) + os.mkdir(dst_folder_file) + + # move file + dst_file_path = os.path.join(dst_folder_file, src_filename + "." + output_filetype) + shutil.copy(src_file, dst_file_path) + + + +# Move the raster to the storage folders +def crop_raster_layers_by_gaul0(): + print "TODO: crop_raster_layers_by_gaul0" + + +# Vector Data +def preprocess_vector_data_for_storage(srs_folder, dst_folder, filetype="shp"): + folders = glob.glob(srs_folder +"/*") + for folder in folders: + if os.path.isdir(folder): + files = glob.glob(folder + "/*." + filetype) + for src_file in files: + path, filename_ext, filename = get_filename(src_file, True) + print path, filename_ext, filename + # get all shapefile reference files + shpfiles = glob.glob(path + "/" + filename + ".*") + _move_vector_files_to_storage(dst_folder, filename.lower(), shpfiles) + +def _move_vector_files_to_storage(dst_folder, dst_filename, src_files): + print "_move_vector_files_to_storage" + + dst_folder_file = os.path.join(dst_folder, dst_filename) + print dst_folder_file + + # create folder + if os.path.isdir(dst_folder_file) is True: + shutil.rmtree(dst_folder_file) + os.mkdir(dst_folder_file) + + # move files + for src_file in src_files: + path, filename_ext, filename = get_filename(src_file, True) + dst_file_path = os.path.join(dst_folder_file, filename_ext) + print dst_file_path + shutil.copy(src_file, dst_file_path) + + + +if __name__ == '__main__': + # Folders + storage_raster_folder = "/home/vortex/Desktop/LAYERS/ghg/process_data/storage/raster" + storage_vector_folder = "/home/vortex/Desktop/LAYERS/ghg/process_data/storage/vector" + process_data_folder = "/home/vortex/Desktop/LAYERS/ghg/process_data" + + # 4326 Raster Data + # preprocess_raster_data_for_storage(process_data_folder + "/4326", storage_raster_folder) + + # 3975 Raster Data + #preprocess_vector_data_for_storage(process_data_folder + "/3975", storage_vector_folder) + + # 3975 Raster Data + #preprocess_vector_data_for_storage(process_data_folder + "/3975", storage_vector_folder) \ No newline at end of file From 22a6f26de65318a131c2467bcb507556e50906a0 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Thu, 7 May 2015 17:49:11 +0200 Subject: [PATCH 28/39] new slds --- ...urning_closedshrublands_fuelbiomass_EN.sld | 22 +++ ...urning_closedshrublands_fuelbiomass_EN.xml | 9 ++ ...urning_extratropical_forests_ch4_ef_EN.sld | 21 +++ ...urning_extratropical_forests_ch4_ef_EN.xml | 9 ++ ...urning_extratropical_forests_co2_ef_EN.sld | 21 +++ ...urning_extratropical_forests_co2_ef_EN.xml | 9 ++ ...urning_extratropical_forests_n2o_ef_EN.sld | 21 +++ ...urning_extratropical_forests_n2o_ef_EN.xml | 9 ++ .../burning_grasslands_fuelbiomass_EN.sld | 22 +++ .../burning_grasslands_fuelbiomass_EN.xml | 9 ++ ...ing_nontropical_forests_fuelbiomass_EN.sld | 22 +++ ...ing_nontropical_forests_fuelbiomass_EN.xml | 9 ++ .../burning_openshrublands_fuelbiomass_EN.sld | 21 +++ .../burning_openshrublands_fuelbiomass_EN.xml | 9 ++ .../burning_peatland_fuelbiomass_EN.sld | 21 +++ .../burning_peatland_fuelbiomass_EN.xml | 9 ++ .../burning_peatlands_ch4_ef_EN.sld | 21 +++ .../burning_peatlands_ch4_ef_EN.xml | 9 ++ .../burning_peatlands_co2_ef_EN.sld | 21 +++ .../burning_peatlands_co2_ef_EN.xml | 9 ++ .../burning_peatlands_n2o_ef_EN.sld | 21 +++ .../burning_peatlands_n2o_ef_EN.xml | 9 ++ .../burning_savanna_fuelbiomass_EN.sld | 22 +++ .../burning_savanna_fuelbiomass_EN.xml | 9 ++ .../burning_savannagrassland_ch4_ef_EN.sld | 21 +++ .../burning_savannagrassland_ch4_ef_EN.xml | 9 ++ .../burning_savannagrassland_co2_ef_EN.sld | 21 +++ .../burning_savannagrassland_co2_ef_EN.xml | 9 ++ .../burning_savannagrassland_n2o_ef_EN.sld | 21 +++ .../burning_savannagrassland_n2o_ef_EN.xml | 9 ++ .../burning_tropical_forests_ch4_ef_EN.sld | 21 +++ .../burning_tropical_forests_ch4_ef_EN.xml | 9 ++ .../burning_tropical_forests_co2_ef_EN.sld | 21 +++ .../burning_tropical_forests_co2_ef_EN.xml | 9 ++ ...urning_tropical_forests_fuelbiomass_EN.sld | 21 +++ ...urning_tropical_forests_fuelbiomass_EN.xml | 9 ++ .../burning_tropical_forests_n2o_ef_EN.sld | 21 +++ .../burning_tropical_forests_n2o_ef_EN.xml | 9 ++ .../burning_woodysavanna_fuelbiomass_EN.sld | 22 +++ .../burning_woodysavanna_fuelbiomass_EN.xml | 9 ++ .../ch4_gfed4ba_emissions_burning_EN.sld | 26 ++++ .../ch4_gfed4ba_emissions_burning_EN.xml | 9 ++ .../co2_gfed4ba_emissions_burning_EN.sld | 25 ++++ .../co2_gfed4ba_emissions_burning_EN.xml | 9 ++ .../cultivation_organicsoils_area_EN.sld | 25 ++++ .../cultivation_organicsoils_area_EN.xml | 9 ++ ...tivation_organicsoils_co2_emissions_EN.sld | 21 +++ ...tivation_organicsoils_co2_emissions_EN.xml | 9 ++ ...tivation_organicsoils_n2o_emissions_EN.sld | 27 ++++ ...tivation_organicsoils_n2o_emissions_EN.xml | 9 ++ .../geoserver_sld/dm_gfed4_burning_EN.sld | 33 +++++ .../geoserver_sld/dm_gfed4_burning_EN.xml | 9 ++ .../geoserver_sld/gez2010_EN.sld | 41 ++++++ .../geoserver_sld/gez2010_EN.xml | 9 ++ .../geoserver_sld/gfed4_burnedarea_EN.sld | 25 ++++ .../geoserver_sld/gfed4_burnedarea_EN.xml | 9 ++ .../geoserver_sld/glc2000_EN.sld | 26 ++++ .../geoserver_sld/glc2000_EN.xml | 9 ++ .../griddedlivestock_heads_EN.sld | 24 ++++ .../griddedlivestock_heads_EN.xml | 9 ++ .../geoserver_sld/hwsd_histosols_area_EN.sld | 25 ++++ .../geoserver_sld/hwsd_histosols_area_EN.xml | 9 ++ .../geoserver_sld/jrc_climate_zones_EN.sld | 32 +++++ .../geoserver_sld/jrc_climate_zones_EN.xml | 9 ++ .../n2o_gfed4ba_emissions_burning_EN.sld | 27 ++++ .../n2o_gfed4ba_emissions_burning_EN.xml | 9 ++ .../{ => old}/burned_areas_EN.sld | 0 .../{ => old}/burned_areas_EN.xml | 0 .../{ => old}/cattle10km_ad_v2_EN.sld | 0 .../{ => old}/cattle10km_ad_v2_EN.xml | 0 .../geoserver_sld/{ => old}/ch4_EN.sld | 0 .../geoserver_sld/{ => old}/ch4_ef_EN.sld | 0 .../geoserver_sld/{ => old}/co2_EN.sld | 0 .../geoserver_sld/{ => old}/co2_ef_EN.sld | 0 ...ltivation_organic_soils_-_croplands_EN.sld | 0 ...ltivation_organic_soils_-_croplands_EN.xml | 0 .../geoserver_sld/{ => old}/dm_EN.sld | 0 .../{ => old}/earthstat_crop_area_EN.sld | 0 .../{ => old}/earthstat_crop_area_EN.xml | 0 .../{ => old}/fuelbiomass_EN.sld | 0 .../global_ecological_zones_gez_EN.sld | 0 .../global_land_cover_glc2000_EN.sld | 0 .../{ => old}/jrc_climate_zone_EN.sld | 0 .../{ => old}/jrc_climate_zone_EN.xml | 0 .../geoserver_sld/{ => old}/modis | 0 .../modis_-_land_cover_type_umd_EN.sld | 0 .../modis_-_land_cover_type_umd_EN.xml | 0 .../geoserver_sld/{ => old}/n2o_EN.sld | 0 .../geoserver_sld/{ => old}/n2o_ef_EN.sld | 0 .../{ => old}/sheep10km_ad_v2_EN.sld | 0 .../{ => old}/sheep10km_ad_v2_EN.xml | 0 .../{ => old}/woodysavanna_fuelbiomass_EN.sld | 0 ...sh_GHG_data.py => publish_GHG_data_old.py} | 0 .../prod/data/ghg/publish/publish_data.py | 136 ++++++++++++++++++ .../prod/data/ghg/publish/publish_sld.py | 16 +++ 95 files changed, 1231 insertions(+) create mode 100644 geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/gez2010_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/gez2010_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/glc2000_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/glc2000_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.xml create mode 100644 geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.sld create mode 100644 geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.xml rename geobricks_data_scripts/geoserver_sld/{ => old}/burned_areas_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/burned_areas_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/cattle10km_ad_v2_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/cattle10km_ad_v2_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/ch4_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/ch4_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/co2_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/co2_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/cultivation_organic_soils_-_croplands_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/cultivation_organic_soils_-_croplands_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/dm_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/earthstat_crop_area_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/earthstat_crop_area_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/global_ecological_zones_gez_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/global_land_cover_glc2000_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/jrc_climate_zone_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/jrc_climate_zone_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/modis (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/modis_-_land_cover_type_umd_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/modis_-_land_cover_type_umd_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/n2o_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/n2o_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/sheep10km_ad_v2_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/sheep10km_ad_v2_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => old}/woodysavanna_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/prod/data/ghg/publish/{publish_GHG_data.py => publish_GHG_data_old.py} (100%) create mode 100644 geobricks_data_scripts/prod/data/ghg/publish/publish_data.py create mode 100644 geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py diff --git a/geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.sld new file mode 100644 index 0000000..2a5cf14 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.sld @@ -0,0 +1,22 @@ + + + ClosedShrublands_FuelBiomass + + ClosedShrublands_FuelBiomass + + + + + grid + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.xml new file mode 100644 index 0000000..1bde618 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.sld new file mode 100644 index 0000000..d8fc5c7 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.sld @@ -0,0 +1,21 @@ + + + ExtraTropical_Forests_CH4_EF + + ExtraTropical_Forests_CH4_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.xml new file mode 100644 index 0000000..e84b14b --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.sld new file mode 100644 index 0000000..b9e12c5 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.sld @@ -0,0 +1,21 @@ + + + ExtraTropical_Forests_CO2_EF + + ExtraTropical_Forests_CO2_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.xml new file mode 100644 index 0000000..a23347d --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.sld new file mode 100644 index 0000000..b9cdfb5 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.sld @@ -0,0 +1,21 @@ + + + ExtraTropical_Forests_N2O_EF + + ExtraTropical_Forests_N2O_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.xml new file mode 100644 index 0000000..953ec85 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.sld new file mode 100644 index 0000000..04c95bb --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.sld @@ -0,0 +1,22 @@ + + + Grasslands_FuelBiomass + + Grasslands_FuelBiomass + + + + + grid + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.xml new file mode 100644 index 0000000..bfb8dfa --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.sld new file mode 100644 index 0000000..ff27d1d --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.sld @@ -0,0 +1,22 @@ + + + NonTropical_Forests_FuelBiomass + + NonTropical_Forests_FuelBiomass + + + + + grid + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.xml new file mode 100644 index 0000000..42c10c6 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.sld new file mode 100644 index 0000000..16c3e78 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.sld @@ -0,0 +1,21 @@ + + + OpenShrublands_FuelBiomass + + OpenShrublands_FuelBiomass + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.xml new file mode 100644 index 0000000..04a9793 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.sld new file mode 100644 index 0000000..d56bd96 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.sld @@ -0,0 +1,21 @@ + + + Peatland_FuelBiomass + + Peatland_FuelBiomass + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.xml new file mode 100644 index 0000000..58e455b --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.sld new file mode 100644 index 0000000..beeb93a --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.sld @@ -0,0 +1,21 @@ + + + Peatlands_CH4_EF + + Peatlands_CH4_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.xml new file mode 100644 index 0000000..5dd47f7 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.sld new file mode 100644 index 0000000..520089b --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.sld @@ -0,0 +1,21 @@ + + + Peatlands_CO2_EF + + Peatlands_CO2_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.xml new file mode 100644 index 0000000..6dd8842 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.sld new file mode 100644 index 0000000..2a9ff4c --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.sld @@ -0,0 +1,21 @@ + + + Peatlands_N2O_EF + + Peatlands_N2O_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.xml new file mode 100644 index 0000000..7324a12 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.sld new file mode 100644 index 0000000..d8bf270 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.sld @@ -0,0 +1,22 @@ + + + Savanna_FuelBiomass + + Savanna_FuelBiomass + + + + + grid + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.xml new file mode 100644 index 0000000..de000ca --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.sld new file mode 100644 index 0000000..3b7e0e2 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.sld @@ -0,0 +1,21 @@ + + + SavannaGrassland_N2O_EF + + SavannaGrassland_N2O_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.xml new file mode 100644 index 0000000..04a7fdd --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.sld new file mode 100644 index 0000000..b4c003f --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.sld @@ -0,0 +1,21 @@ + + + SavannaGrassland_CO2_EF + + SavannaGrassland_CO2_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.xml new file mode 100644 index 0000000..a388d7b --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.sld new file mode 100644 index 0000000..c2fffcf --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.sld @@ -0,0 +1,21 @@ + + + SavannaGrassland_N2O_EF + + SavannaGrassland_N2O_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.xml new file mode 100644 index 0000000..ebc448e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.sld new file mode 100644 index 0000000..a2acd86 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.sld @@ -0,0 +1,21 @@ + + + Tropical_Forests_CH4_EF + + Tropical_Forests_CH4_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.xml new file mode 100644 index 0000000..f8c5bd9 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.sld new file mode 100644 index 0000000..b836621 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.sld @@ -0,0 +1,21 @@ + + + Tropical_Forests_CO2_EF + + Tropical_Forests_CO2_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.xml new file mode 100644 index 0000000..8621c4b --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.sld new file mode 100644 index 0000000..472278d --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.sld @@ -0,0 +1,21 @@ + + + Tropical_Forests_FuelBiomass + + Tropical_Forests_FuelBiomass + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.xml new file mode 100644 index 0000000..aba817c --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.sld new file mode 100644 index 0000000..519a608 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.sld @@ -0,0 +1,21 @@ + + + Tropical_Forests_N2O_EF + + Tropical_Forests_N2O_EF + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.xml new file mode 100644 index 0000000..a5941c0 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.sld new file mode 100644 index 0000000..d1cc23e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.sld @@ -0,0 +1,22 @@ + + + WoodySavanna_FuelBiomass + + WoodySavanna_FuelBiomass + + + + + grid + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.xml new file mode 100644 index 0000000..a9a911d --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.sld b/geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.sld new file mode 100644 index 0000000..b8c4a65 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.sld @@ -0,0 +1,26 @@ + + + CH4_GFED4BA_Emissions + + CH4_GFED4BA_Emissions + + + + + grid + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.xml b/geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.xml new file mode 100644 index 0000000..6754750 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.sld b/geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.sld new file mode 100644 index 0000000..b9e5aa0 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.sld @@ -0,0 +1,25 @@ + + + CO2_GFED4BA_Emissions + + CO2_GFED4BA_Emissions + + + + + grid + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.xml b/geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.xml new file mode 100644 index 0000000..604e716 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.sld b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.sld new file mode 100644 index 0000000..ca98c2e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.sld @@ -0,0 +1,25 @@ + + + Cultivation_OrganicSoils_Area + + Cultivation_OrganicSoils_Area + + + + + grid + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.xml b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.xml new file mode 100644 index 0000000..cfdbfa2 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.sld b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.sld new file mode 100644 index 0000000..615eda8 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.sld @@ -0,0 +1,21 @@ + + + Cultivation_OrganicSoils_CO2_emissions + + Cultivation_OrganicSoils_CO2_emissions + + + + + grid + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.xml b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.xml new file mode 100644 index 0000000..23a966e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.sld b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.sld new file mode 100644 index 0000000..d404bd2 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.sld @@ -0,0 +1,27 @@ + + + Cultivation_OrganicSoils_N2O_emissions + + Cultivation_OrganicSoils_N2O_emissions + + + + + grid + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.xml b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.xml new file mode 100644 index 0000000..08ee378 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.sld b/geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.sld new file mode 100644 index 0000000..1d34143 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.sld @@ -0,0 +1,33 @@ + + + + + + + + DM_GFED4_Burning + + + + + + + grid + + 1 + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.xml b/geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.xml new file mode 100644 index 0000000..52013bb --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/gez2010_EN.sld b/geobricks_data_scripts/geoserver_sld/gez2010_EN.sld new file mode 100644 index 0000000..84e4381 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/gez2010_EN.sld @@ -0,0 +1,41 @@ + + + GEZ2010 + + GEZ2010 + + + + + grid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/gez2010_EN.xml b/geobricks_data_scripts/geoserver_sld/gez2010_EN.xml new file mode 100644 index 0000000..9f586e0 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/gez2010_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.sld b/geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.sld new file mode 100644 index 0000000..28606fc --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.sld @@ -0,0 +1,25 @@ + + + GFED4_BurnedArea + + GFED4_BurnedArea + + + + + grid + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.xml b/geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.xml new file mode 100644 index 0000000..be646c2 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/glc2000_EN.sld b/geobricks_data_scripts/geoserver_sld/glc2000_EN.sld new file mode 100644 index 0000000..00d7aa3 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/glc2000_EN.sld @@ -0,0 +1,26 @@ + + + GLC2000 + + GLC2000 + + + + + grid + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/glc2000_EN.xml b/geobricks_data_scripts/geoserver_sld/glc2000_EN.xml new file mode 100644 index 0000000..1e95454 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/glc2000_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.sld b/geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.sld new file mode 100644 index 0000000..8cf0e48 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.sld @@ -0,0 +1,24 @@ + + + GriddedLivestock_heads + + GriddedLivestock_heads + + + + + grid + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.xml b/geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.xml new file mode 100644 index 0000000..a929b6e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.sld b/geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.sld new file mode 100644 index 0000000..c946237 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.sld @@ -0,0 +1,25 @@ + + + HWSD_histosols_Area + + HWSD_histosols_Area + + + + + grid + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.xml b/geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.xml new file mode 100644 index 0000000..a0a121e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.sld b/geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.sld new file mode 100644 index 0000000..c3441ec --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.sld @@ -0,0 +1,32 @@ + + + JRC_CLIMATE_ZONE + + JRC_CLIMATE_ZONE + + + + + grid + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.xml b/geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.xml new file mode 100644 index 0000000..ea59959 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.sld b/geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.sld new file mode 100644 index 0000000..fcd16ae --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.sld @@ -0,0 +1,27 @@ + + + N2O_GFED4BA_Emissions + + N2O_GFED4BA_Emissions + + + + + grid + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.xml b/geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.xml new file mode 100644 index 0000000..f599e8e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.xml @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/geobricks_data_scripts/geoserver_sld/burned_areas_EN.sld b/geobricks_data_scripts/geoserver_sld/old/burned_areas_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burned_areas_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/burned_areas_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burned_areas_EN.xml b/geobricks_data_scripts/geoserver_sld/old/burned_areas_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burned_areas_EN.xml rename to geobricks_data_scripts/geoserver_sld/old/burned_areas_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.sld b/geobricks_data_scripts/geoserver_sld/old/cattle10km_ad_v2_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/cattle10km_ad_v2_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.xml b/geobricks_data_scripts/geoserver_sld/old/cattle10km_ad_v2_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cattle10km_ad_v2_EN.xml rename to geobricks_data_scripts/geoserver_sld/old/cattle10km_ad_v2_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/ch4_EN.sld b/geobricks_data_scripts/geoserver_sld/old/ch4_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/ch4_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/ch4_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/old/ch4_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/ch4_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/ch4_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/co2_EN.sld b/geobricks_data_scripts/geoserver_sld/old/co2_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/co2_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/co2_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/old/co2_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/co2_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/co2_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.sld b/geobricks_data_scripts/geoserver_sld/old/cultivation_organic_soils_-_croplands_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/cultivation_organic_soils_-_croplands_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.xml b/geobricks_data_scripts/geoserver_sld/old/cultivation_organic_soils_-_croplands_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cultivation_organic_soils_-_croplands_EN.xml rename to geobricks_data_scripts/geoserver_sld/old/cultivation_organic_soils_-_croplands_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/dm_EN.sld b/geobricks_data_scripts/geoserver_sld/old/dm_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/dm_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/dm_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.sld b/geobricks_data_scripts/geoserver_sld/old/earthstat_crop_area_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/earthstat_crop_area_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.xml b/geobricks_data_scripts/geoserver_sld/old/earthstat_crop_area_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/earthstat_crop_area_EN.xml rename to geobricks_data_scripts/geoserver_sld/old/earthstat_crop_area_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/old/fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/global_ecological_zones_gez_EN.sld b/geobricks_data_scripts/geoserver_sld/old/global_ecological_zones_gez_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/global_ecological_zones_gez_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/global_ecological_zones_gez_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/global_land_cover_glc2000_EN.sld b/geobricks_data_scripts/geoserver_sld/old/global_land_cover_glc2000_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/global_land_cover_glc2000_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/global_land_cover_glc2000_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.sld b/geobricks_data_scripts/geoserver_sld/old/jrc_climate_zone_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/jrc_climate_zone_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.xml b/geobricks_data_scripts/geoserver_sld/old/jrc_climate_zone_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/jrc_climate_zone_EN.xml rename to geobricks_data_scripts/geoserver_sld/old/jrc_climate_zone_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/modis b/geobricks_data_scripts/geoserver_sld/old/modis similarity index 100% rename from geobricks_data_scripts/geoserver_sld/modis rename to geobricks_data_scripts/geoserver_sld/old/modis diff --git a/geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.sld b/geobricks_data_scripts/geoserver_sld/old/modis_-_land_cover_type_umd_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/modis_-_land_cover_type_umd_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.xml b/geobricks_data_scripts/geoserver_sld/old/modis_-_land_cover_type_umd_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/modis_-_land_cover_type_umd_EN.xml rename to geobricks_data_scripts/geoserver_sld/old/modis_-_land_cover_type_umd_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/n2o_EN.sld b/geobricks_data_scripts/geoserver_sld/old/n2o_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/n2o_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/n2o_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/old/n2o_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/n2o_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/n2o_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.sld b/geobricks_data_scripts/geoserver_sld/old/sheep10km_ad_v2_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/sheep10km_ad_v2_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.xml b/geobricks_data_scripts/geoserver_sld/old/sheep10km_ad_v2_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/sheep10km_ad_v2_EN.xml rename to geobricks_data_scripts/geoserver_sld/old/sheep10km_ad_v2_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/woodysavanna_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/old/woodysavanna_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/woodysavanna_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/old/woodysavanna_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data_old.py similarity index 100% rename from geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data.py rename to geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data_old.py diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py new file mode 100644 index 0000000..3d77afe --- /dev/null +++ b/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py @@ -0,0 +1,136 @@ +import glob +import os +from geobricks_common.core.log import logger +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder + +log = logger(__file__) + +data_manager = get_data_manager() + +workspace = "ghg" + +def default_import(src_folder, defaultStyle=None): + metadata_json = None + if defaultStyle is not None: + metadata_json = { + "dsd": { + "defaultStyle": defaultStyle + } + } + harvest_folder(data_manager, src_folder, workspace, True, True, True, metadata_json) + + +def import_raster_3857(): + print "Import Raster 3857--------------------" + base_folder = "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/3857_display_only/" + + # CH4 Emissions Burning + default_import(base_folder + "CH4_Emissions_Burning_ClosedShrublands", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_Grasslands", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_HumidTropicalForests", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_OtherForests", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_Peatlands", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_Savanna", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_WoodySavanna", "ch4_gfed4ba_emissions_burning_EN") + + # CO2 Emissions Burning + default_import(base_folder + "CO2_Emissions_Burning_ClosedShrublands", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_Grasslands", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_HumidTropicalForests", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_OtherForests", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_Peatlands", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_Savanna", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_WoodySavanna", "co2_gfed4ba_emissions_burning_EN") + + # CO2_CultivationOrganicSoils (The layers have a json file for the style) + default_import(base_folder + "CO2_CultivationOrganicSoils") + + # CultivationOrganicSoils (The layers have a json file for the style) + default_import(base_folder + "CultivationOrganicSoils") + + # DM + default_import(base_folder + "DM_Burning_ClosedShrublands", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_Grasslands", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_HumidTropicalForests", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_OpenShrublands", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_Peatlands", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_Savanna", "dm_gfed4_burning_EN") + default_import(base_folder + "DM_Burning_WoodySavanna", "dm_gfed4_burning_EN") + + # ExtraTropical_Forests_EF (The layers have a json file for the style) + default_import(base_folder + "ExtraTropical_Forests_EF") + + # FuelBiomass (The layers have a json file for the style) + default_import(base_folder + "FuelBiomass") + + # GEZ2010 + default_import(base_folder + "GEZ2010") + + # GFED4 Burned Areas + default_import(base_folder + "GFED4_BurnedAreas_ClosedShrubland", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_DeciduousBroadleafForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_DeciduousNeedleleafForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_Deforestation", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_EvergreenBroadleafForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_EvergreenNeedleleafForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_Grassland", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_MixedForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_OpenShrubland", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_OtherForest", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_Peatlands", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_Savanna", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_WoodySavanna", "gfed4_burnedarea_EN") + + # GLC2000 + default_import(base_folder + "GLC2000") + + # GriddedLivestock + default_import(base_folder + "GriddedLivestock", "griddedlivestock_heads_EN") + + # HWSD + default_import(base_folder + "HWSD", "hwsd_histosols_area_EN") + + # JRC_Climate + default_import(base_folder + "JRC_Climate", "jrc_climate_zones_EN") + + # N2O + # N.B. "N2O_CultivationOrganicSoils" ha come stile "cultivation_organicsoils_n2o_emissions" + default_import(base_folder + "N2O_CultivationOrganicSoils", "cultivation_organicsoils_n2o_emissions_EN") + default_import(base_folder + "N2O_Emissions_Burning_ClosedShrublands", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_Grasslands", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_HumidTropicalForests", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_OpenShrublands", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_OtherForests", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_Peatlands", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_Savanna", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_Emissions_Burning_WoodySavanna", "n2o_gfed4ba_emissions_burning_EN") + + # Burning Peatlands (The layers have a json file for the style) + default_import(base_folder + "Peatlands_EF") + + # Savanna+Grassland_EF (The layers have a json file for the style) + default_import(base_folder + "Savanna_and_Grassland_EF") + + # Emission Factor (The layers have a json file for the style) + default_import(base_folder + "Tropical_Forests_EF") + + +def import_raster_4326(): + print "Import Raster 4326--------------------" + base_folder = "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/4326/*" + + folders = glob.glob(base_folder) + for folder in folders: + if os.path.isdir(folder): + harvest_folder(data_manager, folder, workspace, False, True, True) + + + +if __name__ == '__main__': + + # import_raster_3857() + + import_raster_4326() diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py new file mode 100644 index 0000000..6e31b41 --- /dev/null +++ b/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py @@ -0,0 +1,16 @@ +from geobricks_geoserver_manager.config.config import config +from geobricks_geoserver_manager.core.geoserver_manager_core import GeoserverManager +from geobricks_common.core.filesystem import get_filename +import glob + + +def import_slds(): + geoserver_manager = GeoserverManager(config) + for path in glob.glob("/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/*.sld"): + with open(path, 'r') as f: + data = f.read() + filename = get_filename(path).lower() + geoserver_manager.publish_style(filename + "_EN", data) + + +#import_slds() From b1ab4caf42ce0399862da379dadf085598bd90b7 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Thu, 7 May 2015 17:49:27 +0200 Subject: [PATCH 29/39] updated scripts --- .../delete/data_manager_delete_all_example.py | 2 +- .../prod/data/ghg/publish/publish_data.py | 125 +++++++++--------- .../prod/data/ghg/publish/publish_sld.py | 19 ++- 3 files changed, 76 insertions(+), 70 deletions(-) diff --git a/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py index 0dfd6a9..fbb274a 100644 --- a/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py @@ -5,7 +5,7 @@ for layer in layers: try: print layer["uid"] - #data_manager.delete(layer["uid"]) + data_manager.delete(layer["uid"]) except Exception, e: print e pass \ No newline at end of file diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py index 3d77afe..7096e34 100644 --- a/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py +++ b/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py @@ -1,7 +1,7 @@ import glob import os from geobricks_common.core.log import logger -from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder log = logger(__file__) @@ -26,22 +26,22 @@ def import_raster_3857(): base_folder = "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/3857_display_only/" # CH4 Emissions Burning - default_import(base_folder + "CH4_Emissions_Burning_ClosedShrublands", "ch4_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CH4_Emissions_Burning_Grasslands", "ch4_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CH4_Emissions_Burning_HumidTropicalForests", "ch4_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CH4_Emissions_Burning_OtherForests", "ch4_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CH4_Emissions_Burning_Peatlands", "ch4_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CH4_Emissions_Burning_Savanna", "ch4_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CH4_Emissions_Burning_WoodySavanna", "ch4_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CH4_Emissions_Burning_ClosedShrublands", "ch4_gfed4ba_emissions_burning_en") + default_import(base_folder + "CH4_Emissions_Burning_Grasslands", "ch4_gfed4ba_emissions_burning_en") + default_import(base_folder + "CH4_Emissions_Burning_HumidTropicalForests", "ch4_gfed4ba_emissions_burning_en") + default_import(base_folder + "CH4_Emissions_Burning_OtherForests", "ch4_gfed4ba_emissions_burning_en") + default_import(base_folder + "CH4_Emissions_Burning_Peatlands", "ch4_gfed4ba_emissions_burning_en") + default_import(base_folder + "CH4_Emissions_Burning_Savanna", "ch4_gfed4ba_emissions_burning_en") + default_import(base_folder + "CH4_Emissions_Burning_WoodySavanna", "ch4_gfed4ba_emissions_burning_en") # CO2 Emissions Burning - default_import(base_folder + "CO2_Emissions_Burning_ClosedShrublands", "co2_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CO2_Emissions_Burning_Grasslands", "co2_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CO2_Emissions_Burning_HumidTropicalForests", "co2_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CO2_Emissions_Burning_OtherForests", "co2_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CO2_Emissions_Burning_Peatlands", "co2_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CO2_Emissions_Burning_Savanna", "co2_gfed4ba_emissions_burning_EN") - default_import(base_folder + "CO2_Emissions_Burning_WoodySavanna", "co2_gfed4ba_emissions_burning_EN") + default_import(base_folder + "CO2_Emissions_Burning_ClosedShrublands", "co2_gfed4ba_emissions_burning_en") + default_import(base_folder + "CO2_Emissions_Burning_Grasslands", "co2_gfed4ba_emissions_burning_en") + default_import(base_folder + "CO2_Emissions_Burning_HumidTropicalForests", "co2_gfed4ba_emissions_burning_en") + default_import(base_folder + "CO2_Emissions_Burning_OtherForests", "co2_gfed4ba_emissions_burning_en") + default_import(base_folder + "CO2_Emissions_Burning_Peatlands", "co2_gfed4ba_emissions_burning_en") + default_import(base_folder + "CO2_Emissions_Burning_Savanna", "co2_gfed4ba_emissions_burning_en") + default_import(base_folder + "CO2_Emissions_Burning_WoodySavanna", "co2_gfed4ba_emissions_burning_en") # CO2_CultivationOrganicSoils (The layers have a json file for the style) default_import(base_folder + "CO2_CultivationOrganicSoils") @@ -50,18 +50,15 @@ def import_raster_3857(): default_import(base_folder + "CultivationOrganicSoils") # DM - default_import(base_folder + "DM_Burning_ClosedShrublands", "dm_gfed4_burning_EN") - default_import(base_folder + "DM_Burning_Grasslands", "dm_gfed4_burning_EN") - default_import(base_folder + "DM_Burning_HumidTropicalForests", "dm_gfed4_burning_EN") - default_import(base_folder + "DM_Burning_OpenShrublands", "dm_gfed4_burning_EN") - default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") - default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_EN") - default_import(base_folder + "DM_Burning_Peatlands", "dm_gfed4_burning_EN") - default_import(base_folder + "DM_Burning_Savanna", "dm_gfed4_burning_EN") - default_import(base_folder + "DM_Burning_WoodySavanna", "dm_gfed4_burning_EN") - - # ExtraTropical_Forests_EF (The layers have a json file for the style) - default_import(base_folder + "ExtraTropical_Forests_EF") + default_import(base_folder + "DM_Burning_ClosedShrublands", "dm_gfed4_burning_en") + default_import(base_folder + "DM_Burning_Grasslands", "dm_gfed4_burning_en") + default_import(base_folder + "DM_Burning_HumidTropicalForests", "dm_gfed4_burning_en") + default_import(base_folder + "DM_Burning_OpenShrublands", "dm_gfed4_burning_en") + default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_en") + default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_en") + default_import(base_folder + "DM_Burning_Peatlands", "dm_gfed4_burning_en") + default_import(base_folder + "DM_Burning_Savanna", "dm_gfed4_burning_en") + default_import(base_folder + "DM_Burning_WoodySavanna", "dm_gfed4_burning_en") # FuelBiomass (The layers have a json file for the style) default_import(base_folder + "FuelBiomass") @@ -70,52 +67,56 @@ def import_raster_3857(): default_import(base_folder + "GEZ2010") # GFED4 Burned Areas - default_import(base_folder + "GFED4_BurnedAreas_ClosedShrubland", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_DeciduousBroadleafForest", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_DeciduousNeedleleafForest", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_Deforestation", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_EvergreenBroadleafForest", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_EvergreenNeedleleafForest", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_Grassland", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_MixedForest", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_OpenShrubland", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_OtherForest", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_Peatlands", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_Savanna", "gfed4_burnedarea_EN") - default_import(base_folder + "GFED4_BurnedAreas_WoodySavanna", "gfed4_burnedarea_EN") + default_import(base_folder + "GFED4_BurnedAreas_ClosedShrubland", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_DeciduousBroadleafForest", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_DeciduousNeedleleafForest", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_Deforestation", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_EvergreenBroadleafForest", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_EvergreenNeedleleafForest", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_Grassland", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_MixedForest", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_OpenShrubland", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_OtherForest", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_Peatlands", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_Savanna", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_WoodySavanna", "gfed4_burnedarea_en") # GLC2000 default_import(base_folder + "GLC2000") # GriddedLivestock - default_import(base_folder + "GriddedLivestock", "griddedlivestock_heads_EN") + default_import(base_folder + "GriddedLivestock", "griddedlivestock_heads_en") # HWSD - default_import(base_folder + "HWSD", "hwsd_histosols_area_EN") + default_import(base_folder + "HWSD", "hwsd_histosols_area_en") # JRC_Climate - default_import(base_folder + "JRC_Climate", "jrc_climate_zones_EN") + default_import(base_folder + "JRC_Climate", "jrc_climate_zones_en") # N2O # N.B. "N2O_CultivationOrganicSoils" ha come stile "cultivation_organicsoils_n2o_emissions" - default_import(base_folder + "N2O_CultivationOrganicSoils", "cultivation_organicsoils_n2o_emissions_EN") - default_import(base_folder + "N2O_Emissions_Burning_ClosedShrublands", "n2o_gfed4ba_emissions_burning_EN") - default_import(base_folder + "N2O_Emissions_Burning_Grasslands", "n2o_gfed4ba_emissions_burning_EN") - default_import(base_folder + "N2O_Emissions_Burning_HumidTropicalForests", "n2o_gfed4ba_emissions_burning_EN") - default_import(base_folder + "N2O_Emissions_Burning_OpenShrublands", "n2o_gfed4ba_emissions_burning_EN") - default_import(base_folder + "N2O_Emissions_Burning_OtherForests", "n2o_gfed4ba_emissions_burning_EN") - default_import(base_folder + "N2O_Emissions_Burning_Peatlands", "n2o_gfed4ba_emissions_burning_EN") - default_import(base_folder + "N2O_Emissions_Burning_Savanna", "n2o_gfed4ba_emissions_burning_EN") - default_import(base_folder + "N2O_Emissions_Burning_WoodySavanna", "n2o_gfed4ba_emissions_burning_EN") + default_import(base_folder + "N2O_CultivationOrganicSoils", "cultivation_organicsoils_n2o_emissions_en") + default_import(base_folder + "N2O_Emissions_Burning_ClosedShrublands", "n2o_gfed4ba_emissions_burning_en") + default_import(base_folder + "N2O_Emissions_Burning_Grasslands", "n2o_gfed4ba_emissions_burning_en") + default_import(base_folder + "N2O_Emissions_Burning_HumidTropicalForests", "n2o_gfed4ba_emissions_burning_en") + default_import(base_folder + "N2O_Emissions_Burning_OpenShrublands", "n2o_gfed4ba_emissions_burning_en") + default_import(base_folder + "N2O_Emissions_Burning_OtherForests", "n2o_gfed4ba_emissions_burning_en") + default_import(base_folder + "N2O_Emissions_Burning_Peatlands", "n2o_gfed4ba_emissions_burning_en") + default_import(base_folder + "N2O_Emissions_Burning_Savanna", "n2o_gfed4ba_emissions_burning_en") + default_import(base_folder + "N2O_Emissions_Burning_WoodySavanna", "n2o_gfed4ba_emissions_burning_en") + + # # Burning Peatlands (The layers have a json file for the style) + # default_import(base_folder + "Peatlands_EF") + # + # # Savanna+Grassland_EF (The layers have a json file for the style) + # default_import(base_folder + "Savanna_and_Grassland_EF") + # + # # Emission Factor (The layers have a json file for the style) + # default_import(base_folder + "Tropical_Forests_EF") + # + # # ExtraTropical_Forests_EF (The layers have a json file for the style) + # default_import(base_folder + "ExtraTropical_Forests_EF") - # Burning Peatlands (The layers have a json file for the style) - default_import(base_folder + "Peatlands_EF") - - # Savanna+Grassland_EF (The layers have a json file for the style) - default_import(base_folder + "Savanna_and_Grassland_EF") - - # Emission Factor (The layers have a json file for the style) - default_import(base_folder + "Tropical_Forests_EF") def import_raster_4326(): @@ -131,6 +132,6 @@ def import_raster_4326(): if __name__ == '__main__': - # import_raster_3857() + import_raster_3857() - import_raster_4326() + # import_raster_4326() diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py index 6e31b41..d702c7d 100644 --- a/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py +++ b/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py @@ -1,16 +1,21 @@ -from geobricks_geoserver_manager.config.config import config -from geobricks_geoserver_manager.core.geoserver_manager_core import GeoserverManager +from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager from geobricks_common.core.filesystem import get_filename import glob def import_slds(): - geoserver_manager = GeoserverManager(config) - for path in glob.glob("/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/*.sld"): + data_manager = get_data_manager() + for path in glob.glob("/home/vortex/repos/FENIX-MAPS/geobricks/geobricks_data_scripts/geobricks_data_scripts/geoserver_sld/*.sld"): with open(path, 'r') as f: data = f.read() - filename = get_filename(path).lower() - geoserver_manager.publish_style(filename + "_EN", data) + stylename = get_filename(path).lower() + # if stylename[-3:] == "_en": + # stylename = stylename[:len(stylename)-3] + "_EN" + overwrite = False + print "Uploading Style: %s (overwrite %s) " % (stylename, overwrite) + data_manager.geoserver_manager.publish_style(stylename, data, overwrite) + #data_manager.geoserver_manager.delete_style(stylename) -#import_slds() + +import_slds() From 7b613e5f35055fb1865b401b5ff6ec8a43cb3928 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Thu, 7 May 2015 17:49:43 +0200 Subject: [PATCH 30/39] updated script --- geobricks_data_scripts/prod/utils/data_manager_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/geobricks_data_scripts/prod/utils/data_manager_util.py b/geobricks_data_scripts/prod/utils/data_manager_util.py index 695fc89..a3e9003 100644 --- a/geobricks_data_scripts/prod/utils/data_manager_util.py +++ b/geobricks_data_scripts/prod/utils/data_manager_util.py @@ -3,4 +3,6 @@ # get the default data manager for demo db def get_data_manager(): + print "Prod DAata Manager" + print settings return DataManager(settings) \ No newline at end of file From 0b7b2be4dcb2f1bc084fb39a27d863aadcef2bd1 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Tue, 12 May 2015 11:24:25 +0200 Subject: [PATCH 31/39] updated slds --- .../Burning_AllForests_CH4_EF.sld | 29 ++++++++++++ .../Burning_AllForests_CO2_EF.sld | 29 ++++++++++++ .../Burning_AllForests_FuelBiomass.sld | 30 ++++++++++++ .../Burning_AllForests_N2O_EF.sld | 29 ++++++++++++ .../Burning_ClosedShrublands_FuelBiomass.sld | 28 +++++++++++ .../Burning_Grasslands_FuelBiomass.sld | 29 ++++++++++++ .../Burning_OpenShrublands_FuelBiomass.sld | 28 +++++++++++ .../Burning_Peatland_FuelBiomass.sld | 28 +++++++++++ .../Burning_Peatlands_CH4_EF.sld | 28 +++++++++++ .../Burning_Peatlands_CO2_EF.sld | 28 +++++++++++ .../Burning_Peatlands_N2O_EF.sld | 28 +++++++++++ ...rning_SavannaGrasslandShrubland_CH4_EF.sld | 28 +++++++++++ ...rning_SavannaGrasslandShrubland_CO2_EF.sld | 28 +++++++++++ ...rning_SavannaGrasslandShrubland_N2O_EF.sld | 28 +++++++++++ .../Burning_Savanna_FuelBiomass.sld | 28 +++++++++++ .../Burning_WoodySavanna_FuelBiomass.sld | 29 ++++++++++++ .../CH4_GFED4BA_Emissions_Burning.sld | 32 +++++++++++++ ...CO2_Emissions_Cultivation_OrganicSoils.sld | 32 +++++++++++++ .../CO2_GFED4BA_Emissions_Burning.sld | 32 +++++++++++++ .../Cultivation_OrganicSoils_Area.sld | 32 +++++++++++++ .../Cultivation_OrganicSoils_N2O_EF.sld | 29 ++++++++++++ .../geoserver_sld/DM_GFED4_Burning.sld | 34 ++++++++++++++ .../geoserver_sld/GEZ2010.sld | 47 +++++++++++++++++++ .../geoserver_sld/GFED4_BurnedArea.sld | 32 +++++++++++++ .../geoserver_sld/GLC2000.sld | 33 +++++++++++++ .../geoserver_sld/GriddedLivestock_heads.sld | 31 ++++++++++++ .../geoserver_sld/HWSD_histosols_Area.sld | 32 +++++++++++++ .../geoserver_sld/JRC_Climate_zones.sld | 38 +++++++++++++++ ...N2O_Emissions_Cultivation_OrganicSoils.sld | 34 ++++++++++++++ .../N2O_GFED4BA_Emissions_Burning.sld | 34 ++++++++++++++ .../burned_areas_EN.sld | 0 .../burned_areas_EN.xml | 0 .../cattle10km_ad_v2_EN.sld | 0 .../cattle10km_ad_v2_EN.xml | 0 .../{old => old_to_delete}/ch4_EN.sld | 0 .../{old => old_to_delete}/ch4_ef_EN.sld | 0 .../{old => old_to_delete}/co2_EN.sld | 0 .../{old => old_to_delete}/co2_ef_EN.sld | 0 ...ltivation_organic_soils_-_croplands_EN.sld | 0 ...ltivation_organic_soils_-_croplands_EN.xml | 0 .../{old => old_to_delete}/dm_EN.sld | 0 .../earthstat_crop_area_EN.sld | 0 .../earthstat_crop_area_EN.xml | 0 .../{old => old_to_delete}/fuelbiomass_EN.sld | 0 .../global_ecological_zones_gez_EN.sld | 0 .../global_land_cover_glc2000_EN.sld | 0 .../jrc_climate_zone_EN.sld | 0 .../jrc_climate_zone_EN.xml | 0 .../{old => old_to_delete}/modis | 0 .../modis_-_land_cover_type_umd_EN.sld | 0 .../modis_-_land_cover_type_umd_EN.xml | 0 .../{old => old_to_delete}/n2o_EN.sld | 0 .../{old => old_to_delete}/n2o_ef_EN.sld | 0 .../sheep10km_ad_v2_EN.sld | 0 .../sheep10km_ad_v2_EN.xml | 0 .../woodysavanna_fuelbiomass_EN.sld | 0 ...urning_closedshrublands_fuelbiomass_EN.sld | 0 ...urning_closedshrublands_fuelbiomass_EN.xml | 0 ...urning_extratropical_forests_ch4_ef_EN.sld | 0 ...urning_extratropical_forests_ch4_ef_EN.xml | 0 ...urning_extratropical_forests_co2_ef_EN.sld | 0 ...urning_extratropical_forests_co2_ef_EN.xml | 0 ...urning_extratropical_forests_n2o_ef_EN.sld | 0 ...urning_extratropical_forests_n2o_ef_EN.xml | 0 .../burning_grasslands_fuelbiomass_EN.sld | 0 .../burning_grasslands_fuelbiomass_EN.xml | 0 ...ing_nontropical_forests_fuelbiomass_EN.sld | 0 ...ing_nontropical_forests_fuelbiomass_EN.xml | 0 .../burning_openshrublands_fuelbiomass_EN.sld | 0 .../burning_openshrublands_fuelbiomass_EN.xml | 0 .../burning_peatland_fuelbiomass_EN.sld | 0 .../burning_peatland_fuelbiomass_EN.xml | 0 .../burning_peatlands_ch4_ef_EN.sld | 0 .../burning_peatlands_ch4_ef_EN.xml | 0 .../burning_peatlands_co2_ef_EN.sld | 0 .../burning_peatlands_co2_ef_EN.xml | 0 .../burning_peatlands_n2o_ef_EN.sld | 0 .../burning_peatlands_n2o_ef_EN.xml | 0 .../burning_savanna_fuelbiomass_EN.sld | 0 .../burning_savanna_fuelbiomass_EN.xml | 0 .../burning_savannagrassland_ch4_ef_EN.sld | 0 .../burning_savannagrassland_ch4_ef_EN.xml | 0 .../burning_savannagrassland_co2_ef_EN.sld | 0 .../burning_savannagrassland_co2_ef_EN.xml | 0 .../burning_savannagrassland_n2o_ef_EN.sld | 0 .../burning_savannagrassland_n2o_ef_EN.xml | 0 .../burning_tropical_forests_ch4_ef_EN.sld | 0 .../burning_tropical_forests_ch4_ef_EN.xml | 0 .../burning_tropical_forests_co2_ef_EN.sld | 0 .../burning_tropical_forests_co2_ef_EN.xml | 0 ...urning_tropical_forests_fuelbiomass_EN.sld | 0 ...urning_tropical_forests_fuelbiomass_EN.xml | 0 .../burning_tropical_forests_n2o_ef_EN.sld | 0 .../burning_tropical_forests_n2o_ef_EN.xml | 0 .../burning_woodysavanna_fuelbiomass_EN.sld | 0 .../burning_woodysavanna_fuelbiomass_EN.xml | 0 .../ch4_gfed4ba_emissions_burning_EN.sld | 0 .../ch4_gfed4ba_emissions_burning_EN.xml | 0 .../co2_gfed4ba_emissions_burning_EN.sld | 0 .../co2_gfed4ba_emissions_burning_EN.xml | 0 .../cultivation_organicsoils_area_EN.sld | 0 .../cultivation_organicsoils_area_EN.xml | 0 ...tivation_organicsoils_co2_emissions_EN.sld | 0 ...tivation_organicsoils_co2_emissions_EN.xml | 0 ...tivation_organicsoils_n2o_emissions_EN.sld | 0 ...tivation_organicsoils_n2o_emissions_EN.xml | 0 .../{ => working}/dm_gfed4_burning_EN.sld | 0 .../{ => working}/dm_gfed4_burning_EN.xml | 0 .../{ => working}/gez2010_EN.sld | 0 .../{ => working}/gez2010_EN.xml | 0 .../{ => working}/gfed4_burnedarea_EN.sld | 0 .../{ => working}/gfed4_burnedarea_EN.xml | 0 .../{ => working}/glc2000_EN.sld | 0 .../{ => working}/glc2000_EN.xml | 0 .../griddedlivestock_heads_EN.sld | 0 .../griddedlivestock_heads_EN.xml | 0 .../{ => working}/hwsd_histosols_area_EN.sld | 0 .../{ => working}/hwsd_histosols_area_EN.xml | 0 .../{ => working}/jrc_climate_zones_EN.sld | 0 .../{ => working}/jrc_climate_zones_EN.xml | 0 .../n2o_gfed4ba_emissions_burning_EN.sld | 0 .../n2o_gfed4ba_emissions_burning_EN.xml | 0 .../prod/data/ghg/prepare_data.py | 1 + 123 files changed, 928 insertions(+) create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_AllForests_CH4_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_AllForests_CO2_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_AllForests_FuelBiomass.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_AllForests_N2O_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_ClosedShrublands_FuelBiomass.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_Grasslands_FuelBiomass.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_OpenShrublands_FuelBiomass.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_Peatland_FuelBiomass.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_Peatlands_CH4_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_Peatlands_CO2_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_Peatlands_N2O_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_CH4_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_CO2_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_N2O_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_Savanna_FuelBiomass.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Burning_WoodySavanna_FuelBiomass.sld create mode 100644 geobricks_data_scripts/geoserver_sld/CH4_GFED4BA_Emissions_Burning.sld create mode 100644 geobricks_data_scripts/geoserver_sld/CO2_Emissions_Cultivation_OrganicSoils.sld create mode 100644 geobricks_data_scripts/geoserver_sld/CO2_GFED4BA_Emissions_Burning.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Cultivation_OrganicSoils_Area.sld create mode 100644 geobricks_data_scripts/geoserver_sld/Cultivation_OrganicSoils_N2O_EF.sld create mode 100644 geobricks_data_scripts/geoserver_sld/DM_GFED4_Burning.sld create mode 100644 geobricks_data_scripts/geoserver_sld/GEZ2010.sld create mode 100644 geobricks_data_scripts/geoserver_sld/GFED4_BurnedArea.sld create mode 100644 geobricks_data_scripts/geoserver_sld/GLC2000.sld create mode 100644 geobricks_data_scripts/geoserver_sld/GriddedLivestock_heads.sld create mode 100644 geobricks_data_scripts/geoserver_sld/HWSD_histosols_Area.sld create mode 100644 geobricks_data_scripts/geoserver_sld/JRC_Climate_zones.sld create mode 100644 geobricks_data_scripts/geoserver_sld/N2O_Emissions_Cultivation_OrganicSoils.sld create mode 100644 geobricks_data_scripts/geoserver_sld/N2O_GFED4BA_Emissions_Burning.sld rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/burned_areas_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/burned_areas_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/cattle10km_ad_v2_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/cattle10km_ad_v2_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/ch4_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/ch4_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/co2_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/co2_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/cultivation_organic_soils_-_croplands_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/cultivation_organic_soils_-_croplands_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/dm_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/earthstat_crop_area_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/earthstat_crop_area_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/global_ecological_zones_gez_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/global_land_cover_glc2000_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/jrc_climate_zone_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/jrc_climate_zone_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/modis (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/modis_-_land_cover_type_umd_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/modis_-_land_cover_type_umd_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/n2o_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/n2o_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/sheep10km_ad_v2_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/sheep10km_ad_v2_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{old => old_to_delete}/woodysavanna_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_burning_closedshrublands_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_burning_closedshrublands_fuelbiomass_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_extratropical_forests_ch4_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_extratropical_forests_ch4_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_extratropical_forests_co2_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_extratropical_forests_co2_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_extratropical_forests_n2o_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_extratropical_forests_n2o_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_grasslands_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_grasslands_fuelbiomass_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_nontropical_forests_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_nontropical_forests_fuelbiomass_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_openshrublands_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_openshrublands_fuelbiomass_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_peatland_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_peatland_fuelbiomass_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_peatlands_ch4_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_peatlands_ch4_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_peatlands_co2_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_peatlands_co2_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_peatlands_n2o_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_peatlands_n2o_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_savanna_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_savanna_fuelbiomass_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_savannagrassland_ch4_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_savannagrassland_ch4_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_savannagrassland_co2_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_savannagrassland_co2_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_savannagrassland_n2o_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_savannagrassland_n2o_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_tropical_forests_ch4_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_tropical_forests_ch4_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_tropical_forests_co2_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_tropical_forests_co2_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_tropical_forests_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_tropical_forests_fuelbiomass_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_tropical_forests_n2o_ef_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_tropical_forests_n2o_ef_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_woodysavanna_fuelbiomass_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/burning_woodysavanna_fuelbiomass_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/ch4_gfed4ba_emissions_burning_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/ch4_gfed4ba_emissions_burning_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/co2_gfed4ba_emissions_burning_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/co2_gfed4ba_emissions_burning_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/cultivation_organicsoils_area_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/cultivation_organicsoils_area_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/cultivation_organicsoils_co2_emissions_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/cultivation_organicsoils_co2_emissions_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/cultivation_organicsoils_n2o_emissions_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/cultivation_organicsoils_n2o_emissions_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/dm_gfed4_burning_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/dm_gfed4_burning_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/gez2010_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/gez2010_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/gfed4_burnedarea_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/gfed4_burnedarea_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/glc2000_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/glc2000_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/griddedlivestock_heads_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/griddedlivestock_heads_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/hwsd_histosols_area_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/hwsd_histosols_area_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/jrc_climate_zones_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/jrc_climate_zones_EN.xml (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/n2o_gfed4ba_emissions_burning_EN.sld (100%) rename geobricks_data_scripts/geoserver_sld/{ => working}/n2o_gfed4ba_emissions_burning_EN.xml (100%) create mode 100644 geobricks_data_scripts/prod/data/ghg/prepare_data.py diff --git a/geobricks_data_scripts/geoserver_sld/Burning_AllForests_CH4_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_AllForests_CH4_EF.sld new file mode 100644 index 0000000..5d2daa3 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_AllForests_CH4_EF.sld @@ -0,0 +1,29 @@ + + + + + + + + Burning_AllForests_CH4_EF + + + + + + + grid + + 1 + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_AllForests_CO2_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_AllForests_CO2_EF.sld new file mode 100644 index 0000000..1c48cc9 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_AllForests_CO2_EF.sld @@ -0,0 +1,29 @@ + + + + + + + + Burning_AllForests_CO2_EF + + + + + + + grid + + 1 + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_AllForests_FuelBiomass.sld b/geobricks_data_scripts/geoserver_sld/Burning_AllForests_FuelBiomass.sld new file mode 100644 index 0000000..d9d1be2 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_AllForests_FuelBiomass.sld @@ -0,0 +1,30 @@ + + + + + + + + AllForests_FuelBiomass + + + + + + + grid + + 1 + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_AllForests_N2O_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_AllForests_N2O_EF.sld new file mode 100644 index 0000000..bf8e497 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_AllForests_N2O_EF.sld @@ -0,0 +1,29 @@ + + + + + + + + Burning_AllForests_N2O_EF + + + + + + + grid + + 1 + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_ClosedShrublands_FuelBiomass.sld b/geobricks_data_scripts/geoserver_sld/Burning_ClosedShrublands_FuelBiomass.sld new file mode 100644 index 0000000..24f3f04 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_ClosedShrublands_FuelBiomass.sld @@ -0,0 +1,28 @@ + + + + + + + + ClosedShrublands_FuelBiomass + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_Grasslands_FuelBiomass.sld b/geobricks_data_scripts/geoserver_sld/Burning_Grasslands_FuelBiomass.sld new file mode 100644 index 0000000..65ca5c5 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_Grasslands_FuelBiomass.sld @@ -0,0 +1,29 @@ + + + + + + + + Grasslands_FuelBiomass + + + + + + + grid + + 1 + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_OpenShrublands_FuelBiomass.sld b/geobricks_data_scripts/geoserver_sld/Burning_OpenShrublands_FuelBiomass.sld new file mode 100644 index 0000000..c3a4048 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_OpenShrublands_FuelBiomass.sld @@ -0,0 +1,28 @@ + + + + + + + + OpenShrublands_FuelBiomass + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_Peatland_FuelBiomass.sld b/geobricks_data_scripts/geoserver_sld/Burning_Peatland_FuelBiomass.sld new file mode 100644 index 0000000..92e9360 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_Peatland_FuelBiomass.sld @@ -0,0 +1,28 @@ + + + + + + + + Peatland_FuelBiomass + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_CH4_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_CH4_EF.sld new file mode 100644 index 0000000..16a024f --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_CH4_EF.sld @@ -0,0 +1,28 @@ + + + + + + + + Peatlands_CH4_EF + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_CO2_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_CO2_EF.sld new file mode 100644 index 0000000..92efa6b --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_CO2_EF.sld @@ -0,0 +1,28 @@ + + + + + + + + Peatlands_CO2_EF + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_N2O_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_N2O_EF.sld new file mode 100644 index 0000000..edda989 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_Peatlands_N2O_EF.sld @@ -0,0 +1,28 @@ + + + + + + + + Peatlands_N2O_EF + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_CH4_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_CH4_EF.sld new file mode 100644 index 0000000..63b3cb1 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_CH4_EF.sld @@ -0,0 +1,28 @@ + + + + + + + + Burning_Savanna_Grassland_CH4_EF + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_CO2_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_CO2_EF.sld new file mode 100644 index 0000000..57ba5c4 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_CO2_EF.sld @@ -0,0 +1,28 @@ + + + + + + + + Burning_SavannaGrasslandShrubland_CO2_EF + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_N2O_EF.sld b/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_N2O_EF.sld new file mode 100644 index 0000000..41784b8 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_SavannaGrasslandShrubland_N2O_EF.sld @@ -0,0 +1,28 @@ + + + + + + + + Burning_SavannaGrasslandSHrubland_N2O_EF + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_Savanna_FuelBiomass.sld b/geobricks_data_scripts/geoserver_sld/Burning_Savanna_FuelBiomass.sld new file mode 100644 index 0000000..d74e8cb --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_Savanna_FuelBiomass.sld @@ -0,0 +1,28 @@ + + + + + + + + Burning_Savanna_FuelBiomass + + + + + + + grid + + 1 + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Burning_WoodySavanna_FuelBiomass.sld b/geobricks_data_scripts/geoserver_sld/Burning_WoodySavanna_FuelBiomass.sld new file mode 100644 index 0000000..885476e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Burning_WoodySavanna_FuelBiomass.sld @@ -0,0 +1,29 @@ + + + + + + + + Burning_WoodySavanna_FuelBiomass + + + + + + + grid + + 1 + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/CH4_GFED4BA_Emissions_Burning.sld b/geobricks_data_scripts/geoserver_sld/CH4_GFED4BA_Emissions_Burning.sld new file mode 100644 index 0000000..e752fd6 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/CH4_GFED4BA_Emissions_Burning.sld @@ -0,0 +1,32 @@ + + + + + + + + CH4_GFED4BA_Emissions + + + + + + + grid + + 1 + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/CO2_Emissions_Cultivation_OrganicSoils.sld b/geobricks_data_scripts/geoserver_sld/CO2_Emissions_Cultivation_OrganicSoils.sld new file mode 100644 index 0000000..a99a6e7 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/CO2_Emissions_Cultivation_OrganicSoils.sld @@ -0,0 +1,32 @@ + + + + + + + + CO2_Emissions_Cultivation_OrganicSoils + + + + + + + grid + + 1 + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/CO2_GFED4BA_Emissions_Burning.sld b/geobricks_data_scripts/geoserver_sld/CO2_GFED4BA_Emissions_Burning.sld new file mode 100644 index 0000000..64f1251 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/CO2_GFED4BA_Emissions_Burning.sld @@ -0,0 +1,32 @@ + + + + + + + + CO2_GFED4BA_Emissions + + + + + + + grid + + 1 + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Cultivation_OrganicSoils_Area.sld b/geobricks_data_scripts/geoserver_sld/Cultivation_OrganicSoils_Area.sld new file mode 100644 index 0000000..46f0932 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Cultivation_OrganicSoils_Area.sld @@ -0,0 +1,32 @@ + + + + + + + + Cultivation_OrganicSoils_Area + + + + + + + grid + + 1 + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/Cultivation_OrganicSoils_N2O_EF.sld b/geobricks_data_scripts/geoserver_sld/Cultivation_OrganicSoils_N2O_EF.sld new file mode 100644 index 0000000..4665f53 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/Cultivation_OrganicSoils_N2O_EF.sld @@ -0,0 +1,29 @@ + + + + + + + + Cultivation_OrganicSoils_N2O_EF + + + + + + + grid + + 1 + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/DM_GFED4_Burning.sld b/geobricks_data_scripts/geoserver_sld/DM_GFED4_Burning.sld new file mode 100644 index 0000000..8706b22 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/DM_GFED4_Burning.sld @@ -0,0 +1,34 @@ + + + + + + + + DM_GFED4_Burning + + + + + + + grid + + 1 + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/GEZ2010.sld b/geobricks_data_scripts/geoserver_sld/GEZ2010.sld new file mode 100644 index 0000000..96d3935 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/GEZ2010.sld @@ -0,0 +1,47 @@ + + + + + + + + GEZ2010 + + + + + + + grid + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/GFED4_BurnedArea.sld b/geobricks_data_scripts/geoserver_sld/GFED4_BurnedArea.sld new file mode 100644 index 0000000..4c33f5b --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/GFED4_BurnedArea.sld @@ -0,0 +1,32 @@ + + + + + + + + GFED4_BurnedArea + + + + + + + grid + + 1 + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/GLC2000.sld b/geobricks_data_scripts/geoserver_sld/GLC2000.sld new file mode 100644 index 0000000..e416a21 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/GLC2000.sld @@ -0,0 +1,33 @@ + + + + + + + + GLC2000 + + + + + + + grid + + 1 + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/GriddedLivestock_heads.sld b/geobricks_data_scripts/geoserver_sld/GriddedLivestock_heads.sld new file mode 100644 index 0000000..e1d8d97 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/GriddedLivestock_heads.sld @@ -0,0 +1,31 @@ + + + + + + + + GriddedLivestock_heads + + + + + + + grid + + 1 + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/HWSD_histosols_Area.sld b/geobricks_data_scripts/geoserver_sld/HWSD_histosols_Area.sld new file mode 100644 index 0000000..f736795 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/HWSD_histosols_Area.sld @@ -0,0 +1,32 @@ + + + + + + + + HWSD_histosols_Area + + + + + + + grid + + 1 + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/JRC_Climate_zones.sld b/geobricks_data_scripts/geoserver_sld/JRC_Climate_zones.sld new file mode 100644 index 0000000..35d551e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/JRC_Climate_zones.sld @@ -0,0 +1,38 @@ + + + + + + + + JRC_CLIMATE_ZONE + + + + + + + grid + + 1 + + + + + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/N2O_Emissions_Cultivation_OrganicSoils.sld b/geobricks_data_scripts/geoserver_sld/N2O_Emissions_Cultivation_OrganicSoils.sld new file mode 100644 index 0000000..a6c7975 --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/N2O_Emissions_Cultivation_OrganicSoils.sld @@ -0,0 +1,34 @@ + + + + + + + + N2O_Emissions_Cultivation_OrganicSoils + + + + + + + grid + + 1 + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/N2O_GFED4BA_Emissions_Burning.sld b/geobricks_data_scripts/geoserver_sld/N2O_GFED4BA_Emissions_Burning.sld new file mode 100644 index 0000000..b6fca0e --- /dev/null +++ b/geobricks_data_scripts/geoserver_sld/N2O_GFED4BA_Emissions_Burning.sld @@ -0,0 +1,34 @@ + + + + + + + + N2O_GFED4BA_Emissions + + + + + + + grid + + 1 + + + + + + + + + + + + + + + + + diff --git a/geobricks_data_scripts/geoserver_sld/old/burned_areas_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/burned_areas_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/burned_areas_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/burned_areas_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/burned_areas_EN.xml b/geobricks_data_scripts/geoserver_sld/old_to_delete/burned_areas_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/burned_areas_EN.xml rename to geobricks_data_scripts/geoserver_sld/old_to_delete/burned_areas_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/old/cattle10km_ad_v2_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/cattle10km_ad_v2_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/cattle10km_ad_v2_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/cattle10km_ad_v2_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/cattle10km_ad_v2_EN.xml b/geobricks_data_scripts/geoserver_sld/old_to_delete/cattle10km_ad_v2_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/cattle10km_ad_v2_EN.xml rename to geobricks_data_scripts/geoserver_sld/old_to_delete/cattle10km_ad_v2_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/old/ch4_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/ch4_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/ch4_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/ch4_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/ch4_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/ch4_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/ch4_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/co2_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/co2_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/co2_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/co2_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/co2_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/co2_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/co2_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/cultivation_organic_soils_-_croplands_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/cultivation_organic_soils_-_croplands_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/cultivation_organic_soils_-_croplands_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/cultivation_organic_soils_-_croplands_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/cultivation_organic_soils_-_croplands_EN.xml b/geobricks_data_scripts/geoserver_sld/old_to_delete/cultivation_organic_soils_-_croplands_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/cultivation_organic_soils_-_croplands_EN.xml rename to geobricks_data_scripts/geoserver_sld/old_to_delete/cultivation_organic_soils_-_croplands_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/old/dm_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/dm_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/dm_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/dm_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/earthstat_crop_area_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/earthstat_crop_area_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/earthstat_crop_area_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/earthstat_crop_area_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/earthstat_crop_area_EN.xml b/geobricks_data_scripts/geoserver_sld/old_to_delete/earthstat_crop_area_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/earthstat_crop_area_EN.xml rename to geobricks_data_scripts/geoserver_sld/old_to_delete/earthstat_crop_area_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/old/fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/global_ecological_zones_gez_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/global_ecological_zones_gez_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/global_ecological_zones_gez_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/global_ecological_zones_gez_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/global_land_cover_glc2000_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/global_land_cover_glc2000_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/global_land_cover_glc2000_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/global_land_cover_glc2000_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/jrc_climate_zone_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/jrc_climate_zone_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/jrc_climate_zone_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/jrc_climate_zone_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/jrc_climate_zone_EN.xml b/geobricks_data_scripts/geoserver_sld/old_to_delete/jrc_climate_zone_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/jrc_climate_zone_EN.xml rename to geobricks_data_scripts/geoserver_sld/old_to_delete/jrc_climate_zone_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/old/modis b/geobricks_data_scripts/geoserver_sld/old_to_delete/modis similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/modis rename to geobricks_data_scripts/geoserver_sld/old_to_delete/modis diff --git a/geobricks_data_scripts/geoserver_sld/old/modis_-_land_cover_type_umd_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/modis_-_land_cover_type_umd_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/modis_-_land_cover_type_umd_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/modis_-_land_cover_type_umd_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/modis_-_land_cover_type_umd_EN.xml b/geobricks_data_scripts/geoserver_sld/old_to_delete/modis_-_land_cover_type_umd_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/modis_-_land_cover_type_umd_EN.xml rename to geobricks_data_scripts/geoserver_sld/old_to_delete/modis_-_land_cover_type_umd_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/old/n2o_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/n2o_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/n2o_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/n2o_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/n2o_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/n2o_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/n2o_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/sheep10km_ad_v2_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/sheep10km_ad_v2_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/sheep10km_ad_v2_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/sheep10km_ad_v2_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/old/sheep10km_ad_v2_EN.xml b/geobricks_data_scripts/geoserver_sld/old_to_delete/sheep10km_ad_v2_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/sheep10km_ad_v2_EN.xml rename to geobricks_data_scripts/geoserver_sld/old_to_delete/sheep10km_ad_v2_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/old/woodysavanna_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/old_to_delete/woodysavanna_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/old/woodysavanna_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/old_to_delete/woodysavanna_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_burning_closedshrublands_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_burning_closedshrublands_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_burning_closedshrublands_fuelbiomass_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_burning_closedshrublands_fuelbiomass_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_burning_closedshrublands_fuelbiomass_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_ch4_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_ch4_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_ch4_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_ch4_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_ch4_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_co2_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_co2_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_co2_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_co2_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_co2_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_n2o_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_n2o_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_n2o_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_extratropical_forests_n2o_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_extratropical_forests_n2o_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_grasslands_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_grasslands_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_grasslands_fuelbiomass_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_grasslands_fuelbiomass_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_grasslands_fuelbiomass_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_nontropical_forests_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_nontropical_forests_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_nontropical_forests_fuelbiomass_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_nontropical_forests_fuelbiomass_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_nontropical_forests_fuelbiomass_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_openshrublands_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_openshrublands_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_openshrublands_fuelbiomass_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_openshrublands_fuelbiomass_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_openshrublands_fuelbiomass_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_peatland_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_peatland_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_peatland_fuelbiomass_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_peatland_fuelbiomass_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_peatland_fuelbiomass_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_peatlands_ch4_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_peatlands_ch4_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_peatlands_ch4_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_peatlands_ch4_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_peatlands_ch4_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_peatlands_co2_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_peatlands_co2_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_peatlands_co2_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_peatlands_co2_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_peatlands_co2_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_peatlands_n2o_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_peatlands_n2o_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_peatlands_n2o_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_peatlands_n2o_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_peatlands_n2o_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_savanna_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_savanna_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_savanna_fuelbiomass_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_savanna_fuelbiomass_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_savanna_fuelbiomass_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_ch4_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_ch4_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_ch4_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_savannagrassland_ch4_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_ch4_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_co2_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_co2_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_co2_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_savannagrassland_co2_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_co2_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_n2o_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_n2o_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_n2o_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_savannagrassland_n2o_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_savannagrassland_n2o_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_ch4_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_ch4_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_ch4_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_tropical_forests_ch4_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_ch4_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_co2_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_co2_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_co2_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_tropical_forests_co2_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_co2_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_fuelbiomass_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_tropical_forests_fuelbiomass_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_fuelbiomass_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_n2o_ef_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_n2o_ef_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_n2o_ef_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_tropical_forests_n2o_ef_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_tropical_forests_n2o_ef_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.sld b/geobricks_data_scripts/geoserver_sld/working/burning_woodysavanna_fuelbiomass_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/burning_woodysavanna_fuelbiomass_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.xml b/geobricks_data_scripts/geoserver_sld/working/burning_woodysavanna_fuelbiomass_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/burning_woodysavanna_fuelbiomass_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/burning_woodysavanna_fuelbiomass_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.sld b/geobricks_data_scripts/geoserver_sld/working/ch4_gfed4ba_emissions_burning_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/ch4_gfed4ba_emissions_burning_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.xml b/geobricks_data_scripts/geoserver_sld/working/ch4_gfed4ba_emissions_burning_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/ch4_gfed4ba_emissions_burning_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/ch4_gfed4ba_emissions_burning_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.sld b/geobricks_data_scripts/geoserver_sld/working/co2_gfed4ba_emissions_burning_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/co2_gfed4ba_emissions_burning_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.xml b/geobricks_data_scripts/geoserver_sld/working/co2_gfed4ba_emissions_burning_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/co2_gfed4ba_emissions_burning_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/co2_gfed4ba_emissions_burning_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.sld b/geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_area_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_area_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.xml b/geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_area_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_area_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_area_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.sld b/geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_co2_emissions_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_co2_emissions_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.xml b/geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_co2_emissions_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_co2_emissions_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_co2_emissions_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.sld b/geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_n2o_emissions_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_n2o_emissions_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.xml b/geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_n2o_emissions_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/cultivation_organicsoils_n2o_emissions_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/cultivation_organicsoils_n2o_emissions_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.sld b/geobricks_data_scripts/geoserver_sld/working/dm_gfed4_burning_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/dm_gfed4_burning_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.xml b/geobricks_data_scripts/geoserver_sld/working/dm_gfed4_burning_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/dm_gfed4_burning_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/dm_gfed4_burning_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/gez2010_EN.sld b/geobricks_data_scripts/geoserver_sld/working/gez2010_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/gez2010_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/gez2010_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/gez2010_EN.xml b/geobricks_data_scripts/geoserver_sld/working/gez2010_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/gez2010_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/gez2010_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.sld b/geobricks_data_scripts/geoserver_sld/working/gfed4_burnedarea_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/gfed4_burnedarea_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.xml b/geobricks_data_scripts/geoserver_sld/working/gfed4_burnedarea_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/gfed4_burnedarea_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/gfed4_burnedarea_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/glc2000_EN.sld b/geobricks_data_scripts/geoserver_sld/working/glc2000_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/glc2000_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/glc2000_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/glc2000_EN.xml b/geobricks_data_scripts/geoserver_sld/working/glc2000_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/glc2000_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/glc2000_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.sld b/geobricks_data_scripts/geoserver_sld/working/griddedlivestock_heads_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/griddedlivestock_heads_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.xml b/geobricks_data_scripts/geoserver_sld/working/griddedlivestock_heads_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/griddedlivestock_heads_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/griddedlivestock_heads_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.sld b/geobricks_data_scripts/geoserver_sld/working/hwsd_histosols_area_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/hwsd_histosols_area_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.xml b/geobricks_data_scripts/geoserver_sld/working/hwsd_histosols_area_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/hwsd_histosols_area_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/hwsd_histosols_area_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.sld b/geobricks_data_scripts/geoserver_sld/working/jrc_climate_zones_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/jrc_climate_zones_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.xml b/geobricks_data_scripts/geoserver_sld/working/jrc_climate_zones_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/jrc_climate_zones_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/jrc_climate_zones_EN.xml diff --git a/geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.sld b/geobricks_data_scripts/geoserver_sld/working/n2o_gfed4ba_emissions_burning_EN.sld similarity index 100% rename from geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.sld rename to geobricks_data_scripts/geoserver_sld/working/n2o_gfed4ba_emissions_burning_EN.sld diff --git a/geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.xml b/geobricks_data_scripts/geoserver_sld/working/n2o_gfed4ba_emissions_burning_EN.xml similarity index 100% rename from geobricks_data_scripts/geoserver_sld/n2o_gfed4ba_emissions_burning_EN.xml rename to geobricks_data_scripts/geoserver_sld/working/n2o_gfed4ba_emissions_burning_EN.xml diff --git a/geobricks_data_scripts/prod/data/ghg/prepare_data.py b/geobricks_data_scripts/prod/data/ghg/prepare_data.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/prod/data/ghg/prepare_data.py @@ -0,0 +1 @@ +__author__ = 'vortex' From 20361737142ac447eb45ea51423a1e4200cf9e05 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Tue, 12 May 2015 11:24:44 +0200 Subject: [PATCH 32/39] updated ghg domain --- .../prod/data/ghg/createZipProducts.py | 174 ++++++++++++++++-- .../delete/data_manager_delete_all_example.py | 7 +- .../prod/data/ghg/prepare_data.py | 24 ++- .../prod/data/ghg/publish/publish_data.py | 159 ++++++++-------- .../prod/data/ghg/publish/publish_sld.py | 4 +- 5 files changed, 269 insertions(+), 99 deletions(-) diff --git a/geobricks_data_scripts/prod/data/ghg/createZipProducts.py b/geobricks_data_scripts/prod/data/ghg/createZipProducts.py index 18ca638..13d99c9 100644 --- a/geobricks_data_scripts/prod/data/ghg/createZipProducts.py +++ b/geobricks_data_scripts/prod/data/ghg/createZipProducts.py @@ -60,13 +60,147 @@ def createGHGProducts(obj): # create zip file createZip(outputFilePath, folders_to_zip, archiveType) + + +def createGHGProductsGlobal(obj): + inputPath = obj["inputPath"] + outputPath = obj["outputPath"] + archiveType = obj["archiveType"] + zips = obj["zippit"] + + # create the zip files + for to_zip in zips: + print to_zip["name"] + outputFilename = to_zip["filename"] + outputFolderName = to_zip["folderName"] + + # TODO: get all country codes (to be used in the folder input construction) + + # TODO add to the path gaul code i.e. 4326/product/faostat/12/product_name.zip + # Create Product + outputFolderPath = os.path.join(outputPath, outputFolderName) + if not os.path.isdir(outputFolderPath): + os.makedirs(outputFolderPath) + + outputFilePath = os.path.join(outputFolderPath, outputFilename) + + # delete file if already exists + if os.path.isfile(outputFilePath): + os.remove(outputFilePath) + + folders_to_zip = [] + for folder in to_zip["folders"]: + # TODO add to the path gaul code i.e. inputPath/12/inputFolder + folders_to_zip.append(os.path.join(inputPath, folder)) + + # create zip file + createZip(outputFilePath, folders_to_zip, archiveType) + + # Definition of the zip -obj = { - "inputPath": "/home/vortex/Desktop/LAYERS/ghg/process_data/test_input/COUNTRIES/", - "outputPath": "/home/vortex/Desktop/LAYERS/ghg/process_data/output/4326/", +objCountry = { + "inputPath": "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem_052015/4326_download/COUNTRIES", + "outputPath": "/home/vortex/Desktop/LAYERS/ghg/download/4326/", "archiveType": "zip", "codingSystem": "faostat", # this is done in this case for each country + "zippit": [ + { + "name": "GFED4 burned areas", + "folderName": "GFED4_burned_areas", + "filename": "GFED4_burned_areas", + "metadataFilesPath": [], + "folders": ["GFED4 burned areas"] + }, + { + "name": "Burned dry matter", + "folderName": "Burned_dry_matter", + "filename": "Burned_dry_matter", + "metadataFilesPath": [], + "folders": ["Burned dry matter"] + }, + { + "name": "Organic soils area", + "folderName": "Organic_soils_area", + "filename": "Organic_soils_area", + "metadataFilesPath": [], + "folders": ["Organic soils area"] + }, + { + "name": "Climate", + "folderName": "Climate", + "filename": "Climate", + "metadataFilesPath": [], + "folders": ["Climate"] + }, + { + "name": "Ecology", + "folderName": "Ecology", + "filename": "Ecology", + "metadataFilesPath": [], + "folders": ["Ecology"] + }, + { + "name": "Livestock", + "folderName": "Livestock", + "filename": "Livestock", + "metadataFilesPath": [], + "folders": ["Livestock"] + }, + { + "name": "Land Cover", + "folderName": "Land_Cover", + "filename": "Land_Cover", + "metadataFilesPath": [], + "folders": ["Land Cover"] + }, + { + "name": "FAOSTAT fire CO2 emission estimates", + "folderName": "FAOSTAT_fire_CO2_emission_estimates", + "filename": "FAOSTAT_fire_CO2_emission_estimates", + "metadataFilesPath": [], + "folders": ["FAOSTAT fire CO2 emission estimates"] + }, + { + "name": "FAOSTAT fire N2O emission estimates", + "folderName": "FAOSTAT_fire_N2O_emission_estimates", + "filename": "FAOSTAT_fire_N2O_emission_estimates", + "metadataFilesPath": [], + "folders": ["FAOSTAT fire N2O emission estimates"] + }, + { + "name": "FAOSTAT fire CH4 emission estimates", + "folderName": "FAOSTAT_fire_CH4_emission_estimates", + "filename": "FAOSTAT_fire_CH4_emission_estimates", + "metadataFilesPath": [], + "folders": ["FAOSTAT fire CH4 emission estimates"] + }, + { + "name": "FAOSTAT cultivation of organic soil CO2 emission estimates", + "folderName": "FAOSTAT_cultivation_of_organic_soil_CO2_emission_estimates", + "filename": "FAOSTAT_cultivation_of_organic_soil_CO2_emission_estimates", + "metadataFilesPath": [], + "folders": ["FAOSTAT cultivation of organic soil CO2 emission estimates"] + }, + { + "name": "FAOSTAT cultivation of organic soil N2O emission estimates", + "folderName": "FAOSTAT_cultivation_of_organic_soil_N2O_emission_estimates", + "filename": "FAOSTAT_cultivation_of_organic_soil_N2O_emission_estimates", + "metadataFilesPath": [], + "folders": ["FAOSTAT cultivation of organic soil N2O emission estimates"] + } + ] +} + + + +objGlobal = { + # "inputPath": "/home/vortex/Desktop/LAYERS/ghg/process_data/test_input/COUNTRIES/", + "inputPath": "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/4326/", + "outputPath": "/home/vortex/Desktop/LAYERS/ghg/process_data/output/4326/", + "archiveType": "zip", + # "codingSystem": "faostat", + # this is done in this case for each country "zippit": [ { "name": "FAOSTAT Fire CO2 emission estimates", @@ -74,9 +208,11 @@ def createGHGProducts(obj): "filename": "co2_emission_estimates", "metadataFilesPath": [], "folders": [ - "CO2_GFED4BA_Emissions_Burning_ClosedShrublands", - "CO2_GFED4BA_Emissions_Burning_OpenShrublands", - ] + # "CO2_GFED4BA_Emissions_Burning_ClosedShrublands", + # "CO2_GFED4BA_Emissions_Burning_OpenShrublands", + "CO2_Emissions_Burning_ClosedShrublands", + "CO2_Emissions_Burning_OpenShrublands", + ] }, { "name": "DM GFED4 burning peatlands", @@ -84,8 +220,9 @@ def createGHGProducts(obj): "filename": "dm_gfed4_burning_peatlands", "metadataFilesPath": [], "folders": [ - "DM_GFED4_Burning_Peatlands", - ] + # "DM_GFED4_Burning_Peatlands", + "DM_Burning_Peatlands", + ] }, { "name": "N2O GFED4BA Emissions Burning Peatlands", @@ -93,8 +230,9 @@ def createGHGProducts(obj): "filename": "n2o_gfed4ba_emissions_burning_peatlands", "metadataFilesPath": [], "folders": [ - "N2O_GFED4BA_Emissions_Burning_Peatlands", - ] + # "N2O_GFED4BA_Emissions_Burning_Peatlands", + "N2O_Emissions_Burning_Peatlands", + ] }, { "name": "GFED4_BurnedArea_Peatlands", @@ -102,14 +240,20 @@ def createGHGProducts(obj): "filename": "gfed4_burnedarea_peatlands", "metadataFilesPath": [], "folders": [ - "GFED4_BurnedArea_Peatlands", - ] + # "GFED4_BurnedArea_Peatlands", + "GFED4_BurnedAreas_Peatlands", + ] } ] } # remove thte output folder and create new output zip files -if os.path.isdir(obj["outputPath"]): - shutil.rmtree(obj["outputPath"]) -createGHGProducts(obj) \ No newline at end of file +if os.path.isdir(objCountry["outputPath"]): + shutil.rmtree(objCountry["outputPath"]) + +print "Creating by contry products" +createGHGProducts(objCountry) + +# print "Creating by global products" +# createGHGProductsGlobal(objGlobal) \ No newline at end of file diff --git a/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py index fbb274a..103ee07 100644 --- a/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py @@ -4,8 +4,11 @@ layers = data_manager.get_all_layers() for layer in layers: try: - print layer["uid"] - data_manager.delete(layer["uid"]) + #print layer["uid"] + #data_manager.delete(layer["uid"]) + if "gfed4_burnedarea_humidtropicalforests_" in layer["uid"]: + print layer["uid"] + #data_manager.delete(layer["uid"]) except Exception, e: print e pass \ No newline at end of file diff --git a/geobricks_data_scripts/prod/data/ghg/prepare_data.py b/geobricks_data_scripts/prod/data/ghg/prepare_data.py index 22d574d..ef52c38 100644 --- a/geobricks_data_scripts/prod/data/ghg/prepare_data.py +++ b/geobricks_data_scripts/prod/data/ghg/prepare_data.py @@ -1 +1,23 @@ -__author__ = 'vortex' + +from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager + +# data_manager = get_data_manager() +# layers = data_manager.get_all_layers() +# for layer in layers: +# try: +# +# print layer["uid"], layer["meContent"]["seCoverage"]["coverageSectors"]["codes"][0] +# #data_manager.delete(layer["uid"]) +# except Exception, e: +# print e +# pass + + +import glob +import os +folders = glob.glob("/home/vortex/Desktop/LAYERS/ghg/download/4326/Burned_dry_matter/faostat/*") +countries = [] +for f in folders: + countries.append(os.path.basename(os.path.normpath(f))) + +print countries \ No newline at end of file diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py index 7096e34..6fab038 100644 --- a/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py +++ b/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py @@ -25,85 +25,86 @@ def import_raster_3857(): print "Import Raster 3857--------------------" base_folder = "/home/vortex/Desktop/LAYERS/ghg/geodata_handedoverto_simonem/3857_display_only/" - # CH4 Emissions Burning - default_import(base_folder + "CH4_Emissions_Burning_ClosedShrublands", "ch4_gfed4ba_emissions_burning_en") - default_import(base_folder + "CH4_Emissions_Burning_Grasslands", "ch4_gfed4ba_emissions_burning_en") - default_import(base_folder + "CH4_Emissions_Burning_HumidTropicalForests", "ch4_gfed4ba_emissions_burning_en") - default_import(base_folder + "CH4_Emissions_Burning_OtherForests", "ch4_gfed4ba_emissions_burning_en") - default_import(base_folder + "CH4_Emissions_Burning_Peatlands", "ch4_gfed4ba_emissions_burning_en") - default_import(base_folder + "CH4_Emissions_Burning_Savanna", "ch4_gfed4ba_emissions_burning_en") - default_import(base_folder + "CH4_Emissions_Burning_WoodySavanna", "ch4_gfed4ba_emissions_burning_en") - - # CO2 Emissions Burning - default_import(base_folder + "CO2_Emissions_Burning_ClosedShrublands", "co2_gfed4ba_emissions_burning_en") - default_import(base_folder + "CO2_Emissions_Burning_Grasslands", "co2_gfed4ba_emissions_burning_en") - default_import(base_folder + "CO2_Emissions_Burning_HumidTropicalForests", "co2_gfed4ba_emissions_burning_en") - default_import(base_folder + "CO2_Emissions_Burning_OtherForests", "co2_gfed4ba_emissions_burning_en") - default_import(base_folder + "CO2_Emissions_Burning_Peatlands", "co2_gfed4ba_emissions_burning_en") - default_import(base_folder + "CO2_Emissions_Burning_Savanna", "co2_gfed4ba_emissions_burning_en") - default_import(base_folder + "CO2_Emissions_Burning_WoodySavanna", "co2_gfed4ba_emissions_burning_en") - - # CO2_CultivationOrganicSoils (The layers have a json file for the style) - default_import(base_folder + "CO2_CultivationOrganicSoils") - - # CultivationOrganicSoils (The layers have a json file for the style) - default_import(base_folder + "CultivationOrganicSoils") - - # DM - default_import(base_folder + "DM_Burning_ClosedShrublands", "dm_gfed4_burning_en") - default_import(base_folder + "DM_Burning_Grasslands", "dm_gfed4_burning_en") - default_import(base_folder + "DM_Burning_HumidTropicalForests", "dm_gfed4_burning_en") - default_import(base_folder + "DM_Burning_OpenShrublands", "dm_gfed4_burning_en") - default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_en") - default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_en") - default_import(base_folder + "DM_Burning_Peatlands", "dm_gfed4_burning_en") - default_import(base_folder + "DM_Burning_Savanna", "dm_gfed4_burning_en") - default_import(base_folder + "DM_Burning_WoodySavanna", "dm_gfed4_burning_en") - - # FuelBiomass (The layers have a json file for the style) - default_import(base_folder + "FuelBiomass") - - # GEZ2010 - default_import(base_folder + "GEZ2010") - - # GFED4 Burned Areas - default_import(base_folder + "GFED4_BurnedAreas_ClosedShrubland", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_DeciduousBroadleafForest", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_DeciduousNeedleleafForest", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_Deforestation", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_EvergreenBroadleafForest", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_EvergreenNeedleleafForest", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_Grassland", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_MixedForest", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_OpenShrubland", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_OtherForest", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_Peatlands", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_Savanna", "gfed4_burnedarea_en") - default_import(base_folder + "GFED4_BurnedAreas_WoodySavanna", "gfed4_burnedarea_en") - - # GLC2000 - default_import(base_folder + "GLC2000") - - # GriddedLivestock - default_import(base_folder + "GriddedLivestock", "griddedlivestock_heads_en") - - # HWSD - default_import(base_folder + "HWSD", "hwsd_histosols_area_en") - - # JRC_Climate - default_import(base_folder + "JRC_Climate", "jrc_climate_zones_en") - - # N2O - # N.B. "N2O_CultivationOrganicSoils" ha come stile "cultivation_organicsoils_n2o_emissions" - default_import(base_folder + "N2O_CultivationOrganicSoils", "cultivation_organicsoils_n2o_emissions_en") - default_import(base_folder + "N2O_Emissions_Burning_ClosedShrublands", "n2o_gfed4ba_emissions_burning_en") - default_import(base_folder + "N2O_Emissions_Burning_Grasslands", "n2o_gfed4ba_emissions_burning_en") - default_import(base_folder + "N2O_Emissions_Burning_HumidTropicalForests", "n2o_gfed4ba_emissions_burning_en") - default_import(base_folder + "N2O_Emissions_Burning_OpenShrublands", "n2o_gfed4ba_emissions_burning_en") - default_import(base_folder + "N2O_Emissions_Burning_OtherForests", "n2o_gfed4ba_emissions_burning_en") - default_import(base_folder + "N2O_Emissions_Burning_Peatlands", "n2o_gfed4ba_emissions_burning_en") - default_import(base_folder + "N2O_Emissions_Burning_Savanna", "n2o_gfed4ba_emissions_burning_en") - default_import(base_folder + "N2O_Emissions_Burning_WoodySavanna", "n2o_gfed4ba_emissions_burning_en") + # # CH4 Emissions Burning + # default_import(base_folder + "CH4_Emissions_Burning_ClosedShrublands", "ch4_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CH4_Emissions_Burning_Grasslands", "ch4_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CH4_Emissions_Burning_HumidTropicalForests", "ch4_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CH4_Emissions_Burning_OtherForests", "ch4_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CH4_Emissions_Burning_Peatlands", "ch4_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CH4_Emissions_Burning_Savanna", "ch4_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CH4_Emissions_Burning_WoodySavanna", "ch4_gfed4ba_emissions_burning_en") + # + # # CO2 Emissions Burning + # default_import(base_folder + "CO2_Emissions_Burning_ClosedShrublands", "co2_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CO2_Emissions_Burning_Grasslands", "co2_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CO2_Emissions_Burning_HumidTropicalForests", "co2_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CO2_Emissions_Burning_OtherForests", "co2_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CO2_Emissions_Burning_Peatlands", "co2_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CO2_Emissions_Burning_Savanna", "co2_gfed4ba_emissions_burning_en") + # default_import(base_folder + "CO2_Emissions_Burning_WoodySavanna", "co2_gfed4ba_emissions_burning_en") + # + # # CO2_CultivationOrganicSoils (The layers have a json file for the style) + # default_import(base_folder + "CO2_CultivationOrganicSoils") + # + # # CultivationOrganicSoils (The layers have a json file for the style) + # default_import(base_folder + "CultivationOrganicSoils") + # + # # DM + # default_import(base_folder + "DM_Burning_ClosedShrublands", "dm_gfed4_burning_en") + # default_import(base_folder + "DM_Burning_Grasslands", "dm_gfed4_burning_en") + # default_import(base_folder + "DM_Burning_HumidTropicalForests", "dm_gfed4_burning_en") + # default_import(base_folder + "DM_Burning_OpenShrublands", "dm_gfed4_burning_en") + # default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_en") + # default_import(base_folder + "DM_Burning_OtherForests", "dm_gfed4_burning_en") + # default_import(base_folder + "DM_Burning_Peatlands", "dm_gfed4_burning_en") + # default_import(base_folder + "DM_Burning_Savanna", "dm_gfed4_burning_en") + # default_import(base_folder + "DM_Burning_WoodySavanna", "dm_gfed4_burning_en") + # + # # FuelBiomass (The layers have a json file for the style) + # default_import(base_folder + "FuelBiomass") + # + # # GEZ2010 + # default_import(base_folder + "GEZ2010") + # + # # GFED4 Burned Areas + # default_import(base_folder + "GFED4_BurnedAreas_ClosedShrubland", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_DeciduousBroadleafForest", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_DeciduousNeedleleafForest", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_Deforestation", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_EvergreenBroadleafForest", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_EvergreenNeedleleafForest", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_Grassland", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_MixedForest", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_OpenShrubland", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_OtherForest", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_Peatlands", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_Savanna", "gfed4_burnedarea_en") + # default_import(base_folder + "GFED4_BurnedAreas_WoodySavanna", "gfed4_burnedarea_en") + default_import(base_folder + "GFED4_BurnedAreas_HumidTropicalForests", "gfed4_burnedarea_en") + # + # # GLC2000 + # default_import(base_folder + "GLC2000") + # + # # GriddedLivestock + # default_import(base_folder + "GriddedLivestock", "griddedlivestock_heads_en") + # + # # HWSD + # default_import(base_folder + "HWSD", "hwsd_histosols_area_en") + # + # # JRC_Climate + # default_import(base_folder + "JRC_Climate", "jrc_climate_zones_en") + # + # # N2O + # # N.B. "N2O_CultivationOrganicSoils" ha come stile "cultivation_organicsoils_n2o_emissions" + # default_import(base_folder + "N2O_CultivationOrganicSoils", "cultivation_organicsoils_n2o_emissions_en") + # default_import(base_folder + "N2O_Emissions_Burning_ClosedShrublands", "n2o_gfed4ba_emissions_burning_en") + # default_import(base_folder + "N2O_Emissions_Burning_Grasslands", "n2o_gfed4ba_emissions_burning_en") + # default_import(base_folder + "N2O_Emissions_Burning_HumidTropicalForests", "n2o_gfed4ba_emissions_burning_en") + # default_import(base_folder + "N2O_Emissions_Burning_OpenShrublands", "n2o_gfed4ba_emissions_burning_en") + # default_import(base_folder + "N2O_Emissions_Burning_OtherForests", "n2o_gfed4ba_emissions_burning_en") + # default_import(base_folder + "N2O_Emissions_Burning_Peatlands", "n2o_gfed4ba_emissions_burning_en") + # default_import(base_folder + "N2O_Emissions_Burning_Savanna", "n2o_gfed4ba_emissions_burning_en") + # default_import(base_folder + "N2O_Emissions_Burning_WoodySavanna", "n2o_gfed4ba_emissions_burning_en") # # Burning Peatlands (The layers have a json file for the style) # default_import(base_folder + "Peatlands_EF") diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py b/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py index d702c7d..6683861 100644 --- a/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py +++ b/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py @@ -9,8 +9,8 @@ def import_slds(): with open(path, 'r') as f: data = f.read() stylename = get_filename(path).lower() - # if stylename[-3:] == "_en": - # stylename = stylename[:len(stylename)-3] + "_EN" + if stylename[-3:] != "_en": + stylename += "_en" overwrite = False print "Uploading Style: %s (overwrite %s) " % (stylename, overwrite) From 8a3d85996d0c7b085d2b2e8e8423f98b7bf0d745 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Tue, 12 May 2015 11:24:54 +0200 Subject: [PATCH 33/39] print --- geobricks_data_scripts/utils/harvest/harvest_rasters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geobricks_data_scripts/utils/harvest/harvest_rasters.py b/geobricks_data_scripts/utils/harvest/harvest_rasters.py index 82ab755..740a426 100644 --- a/geobricks_data_scripts/utils/harvest/harvest_rasters.py +++ b/geobricks_data_scripts/utils/harvest/harvest_rasters.py @@ -99,7 +99,7 @@ def parse_filename(filename, map_projection_code, product_code=None): # it is gived by the product (TOdO: should it be common by all the layers?) defaultStyle = product_code + "_" + language.upper() - print "-------------" + print "Harvest raster:" print title, product_code, product_label, layerName, defaultStyle, fromdate, todate, prj metadata = { From 826779b1264cb2f3deaad680090a610551bdd4fd Mon Sep 17 00:00:00 2001 From: simonefenix Date: Tue, 12 May 2015 11:25:03 +0200 Subject: [PATCH 34/39] updated slds --- .../Burning_OpenShrublands_FuelBiomass.sld | 49 +++++++++---------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/geobricks_data_scripts/geoserver_sld/Burning_OpenShrublands_FuelBiomass.sld b/geobricks_data_scripts/geoserver_sld/Burning_OpenShrublands_FuelBiomass.sld index c3a4048..f2a3d2f 100644 --- a/geobricks_data_scripts/geoserver_sld/Burning_OpenShrublands_FuelBiomass.sld +++ b/geobricks_data_scripts/geoserver_sld/Burning_OpenShrublands_FuelBiomass.sld @@ -1,28 +1,23 @@ - - - - - - - - OpenShrublands_FuelBiomass - - - - - - - grid - - 1 - - - - - - - - - - + + + OpenShrublands_FuelBiomass + + OpenShrublands_FuelBiomass + + + + + grid + + + + + + + + + + + + From 812a9a455bcc8b726303afa4bc626cefd2e39455 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Tue, 9 Jun 2015 11:17:37 +0200 Subject: [PATCH 35/39] refactoring --- .../delete/data_manager_delete_all_example.py | 11 ---- .../data/ghg/publish/publish_GHG_data.py | 63 ------------------- .../prod/data/ghg/__init__.py | 1 - .../prod/data/ghg/delete/__init__.py | 1 - .../prod/data/ghg/publish/__init__.py | 1 - geobricks_data_scripts/prod/fenix/README.md | 0 geobricks_data_scripts/prod/fenix/__init__.py | 4 ++ geobricks_data_scripts/prod/ghg/__init__.py | 4 ++ .../ghg/check_consistency}/__init__.py | 0 .../check_consistency/check_consistency.py | 2 +- .../prod/{ => ghg}/config/.gitignore | 0 .../ghg/delete => prod/ghg/data}/__init__.py | 0 .../ghg/data/earthstat}/__init__.py | 0 .../data/earthstat/delete}/__init__.py | 0 .../earthstat/delete/delete_all_earthstat.py | 2 +- .../data/earthstat/publish}/__init__.py | 0 .../data/earthstat/publish/publish_harvest.py | 2 +- .../prod/{data => ghg/data/ghg}/__init__.py | 0 .../{ => ghg}/data/ghg/createZipProducts.py | 0 .../data/ghg/delete}/__init__.py | 0 .../delete/data_manager_delete_all_example.py | 2 +- .../prod/{ => ghg}/data/ghg/prepare_data.py | 2 +- .../data/ghg/publish}/__init__.py | 0 .../data/ghg/publish/publish_GHG_data_old.py | 2 +- .../data/ghg/publish/publish_data.py | 2 +- .../{ => ghg}/data/ghg/publish/publish_sld.py | 2 +- .../prod/{ => ghg}/utils/.gitignore | 0 .../publish => ghg/utils}/__init__.py | 0 .../prod/{ => ghg}/utils/data_manager_util.py | 4 +- geobricks_data_scripts/prod/utils/__init__.py | 1 - 30 files changed, 18 insertions(+), 88 deletions(-) delete mode 100644 geobricks_data_scripts/demo/fenix/data/ghg/delete/data_manager_delete_all_example.py delete mode 100644 geobricks_data_scripts/demo/fenix/data/ghg/publish/publish_GHG_data.py delete mode 100644 geobricks_data_scripts/prod/data/ghg/__init__.py delete mode 100644 geobricks_data_scripts/prod/data/ghg/delete/__init__.py delete mode 100644 geobricks_data_scripts/prod/data/ghg/publish/__init__.py create mode 100644 geobricks_data_scripts/prod/fenix/README.md create mode 100644 geobricks_data_scripts/prod/fenix/__init__.py create mode 100644 geobricks_data_scripts/prod/ghg/__init__.py rename geobricks_data_scripts/{demo/fenix/data/ghg => prod/ghg/check_consistency}/__init__.py (100%) rename geobricks_data_scripts/prod/{ => ghg}/check_consistency/check_consistency.py (69%) rename geobricks_data_scripts/prod/{ => ghg}/config/.gitignore (100%) rename geobricks_data_scripts/{demo/fenix/data/ghg/delete => prod/ghg/data}/__init__.py (100%) rename geobricks_data_scripts/{demo/fenix/data/ghg/publish => prod/ghg/data/earthstat}/__init__.py (100%) rename geobricks_data_scripts/prod/{check_consistency => ghg/data/earthstat/delete}/__init__.py (100%) rename geobricks_data_scripts/prod/{ => ghg}/data/earthstat/delete/delete_all_earthstat.py (77%) rename geobricks_data_scripts/prod/{config => ghg/data/earthstat/publish}/__init__.py (100%) rename geobricks_data_scripts/prod/{ => ghg}/data/earthstat/publish/publish_harvest.py (84%) rename geobricks_data_scripts/prod/{data => ghg/data/ghg}/__init__.py (100%) rename geobricks_data_scripts/prod/{ => ghg}/data/ghg/createZipProducts.py (100%) rename geobricks_data_scripts/prod/{data/earthstat => ghg/data/ghg/delete}/__init__.py (100%) rename geobricks_data_scripts/prod/{ => ghg}/data/ghg/delete/data_manager_delete_all_example.py (81%) rename geobricks_data_scripts/prod/{ => ghg}/data/ghg/prepare_data.py (86%) rename geobricks_data_scripts/prod/{data/earthstat/delete => ghg/data/ghg/publish}/__init__.py (100%) rename geobricks_data_scripts/prod/{ => ghg}/data/ghg/publish/publish_GHG_data_old.py (96%) rename geobricks_data_scripts/prod/{ => ghg}/data/ghg/publish/publish_data.py (98%) rename geobricks_data_scripts/prod/{ => ghg}/data/ghg/publish/publish_sld.py (89%) rename geobricks_data_scripts/prod/{ => ghg}/utils/.gitignore (100%) rename geobricks_data_scripts/prod/{data/earthstat/publish => ghg/utils}/__init__.py (100%) rename geobricks_data_scripts/prod/{ => ghg}/utils/data_manager_util.py (66%) delete mode 100644 geobricks_data_scripts/prod/utils/__init__.py diff --git a/geobricks_data_scripts/demo/fenix/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/demo/fenix/data/ghg/delete/data_manager_delete_all_example.py deleted file mode 100644 index 6092615..0000000 --- a/geobricks_data_scripts/demo/fenix/data/ghg/delete/data_manager_delete_all_example.py +++ /dev/null @@ -1,11 +0,0 @@ -from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager - -data_manager = get_data_manager() -layers = data_manager.get_all_layers() -for layer in layers: - try: - print layer["uid"] - data_manager.delete(layer["uid"]) - except Exception, e: - print e - pass \ No newline at end of file diff --git a/geobricks_data_scripts/demo/fenix/data/ghg/publish/publish_GHG_data.py b/geobricks_data_scripts/demo/fenix/data/ghg/publish/publish_GHG_data.py deleted file mode 100644 index 4075bef..0000000 --- a/geobricks_data_scripts/demo/fenix/data/ghg/publish/publish_GHG_data.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import glob - -from geobricks_common.core.log import logger -from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager -from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder - - -log = logger(__file__) - -data_manager = get_data_manager() - -workspace = "ghg" - - -def default_import(src_folder): - log.info("GLC2000: " + src_folder) - metadatas = harvest_folder(data_manager, src_folder, workspace, True, True, True) - - -def burned_areas(src_folder): - folders = glob.glob(os.path.join(src_folder, "*")) - for folder in folders: - folder = os.path.join(folder, "output") - log.info(folder) - metadatas = harvest_folder(data_manager, folder, workspace, True, True, True) - log.info(metadatas) - for metadata in metadatas: - log.info(metadata) - if "3857" in metadata["dsd"]["layerName"]: - log.info("here") - # log.info"metadata["dsd"]["datasource"]) - data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], workspace, "burned_areas_EN") - - -# used just to set the style to modis that was wrong -def set_style_modis(): - data_manager.geoserver_manager.set_style("modis_land_cover_mod12q1_umd_500m_2009_3857", workspace, "modis_-_land_cover_type_umd_EN") - - -if __name__ == '__main__': - base_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/TO_PUBLISH" - - #Cultivation organic soils - default_import(os.path.join(base_folder, "Cultivation_organic_soils_croplands", "Cultivation_organic_soils_-_croplands")) - - # Burned areas - burned_areas(os.path.join(base_folder, "GFED4_BURNEDAREAS_BY_LANDCOVER")) - - # GLC 2000 (TODO: set the style) - #default_import(os.path.join(base_folder, "Global_Land_Cover_2000_(GLC_2000)", "Global_Land_Cover_2000_(GLC_2000)")) - - # griddedlivestock - default_import(os.path.join(base_folder, "GriddedLivestock", "gridded_livestock_of_the_world_v_201")) - - # JRC Climate zone - default_import(os.path.join(base_folder, "JRC_CLIMATE_ZONE", "JRC_climate_zone")) - - # MODIS_-_Land_Cover_type_UMD - default_import(os.path.join(base_folder, "MODIS_land_cover", "MODIS_-_Land_Cover_type_UMD")) - - - diff --git a/geobricks_data_scripts/prod/data/ghg/__init__.py b/geobricks_data_scripts/prod/data/ghg/__init__.py deleted file mode 100644 index 22d574d..0000000 --- a/geobricks_data_scripts/prod/data/ghg/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'vortex' diff --git a/geobricks_data_scripts/prod/data/ghg/delete/__init__.py b/geobricks_data_scripts/prod/data/ghg/delete/__init__.py deleted file mode 100644 index 22d574d..0000000 --- a/geobricks_data_scripts/prod/data/ghg/delete/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'vortex' diff --git a/geobricks_data_scripts/prod/data/ghg/publish/__init__.py b/geobricks_data_scripts/prod/data/ghg/publish/__init__.py deleted file mode 100644 index 22d574d..0000000 --- a/geobricks_data_scripts/prod/data/ghg/publish/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'vortex' diff --git a/geobricks_data_scripts/prod/fenix/README.md b/geobricks_data_scripts/prod/fenix/README.md new file mode 100644 index 0000000..e69de29 diff --git a/geobricks_data_scripts/prod/fenix/__init__.py b/geobricks_data_scripts/prod/fenix/__init__.py new file mode 100644 index 0000000..232e348 --- /dev/null +++ b/geobricks_data_scripts/prod/fenix/__init__.py @@ -0,0 +1,4 @@ +_package__ = 'geobricks_data_scripts' +__author__ = 'Barbaglia, Guido - Murzilli, Simone' +__email__ = 'guido.barbaglia@gmail.com; simone.murzilli@gmail.com;' +__license__ = 'GPL2' diff --git a/geobricks_data_scripts/prod/ghg/__init__.py b/geobricks_data_scripts/prod/ghg/__init__.py new file mode 100644 index 0000000..232e348 --- /dev/null +++ b/geobricks_data_scripts/prod/ghg/__init__.py @@ -0,0 +1,4 @@ +_package__ = 'geobricks_data_scripts' +__author__ = 'Barbaglia, Guido - Murzilli, Simone' +__email__ = 'guido.barbaglia@gmail.com; simone.murzilli@gmail.com;' +__license__ = 'GPL2' diff --git a/geobricks_data_scripts/demo/fenix/data/ghg/__init__.py b/geobricks_data_scripts/prod/ghg/check_consistency/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/fenix/data/ghg/__init__.py rename to geobricks_data_scripts/prod/ghg/check_consistency/__init__.py diff --git a/geobricks_data_scripts/prod/check_consistency/check_consistency.py b/geobricks_data_scripts/prod/ghg/check_consistency/check_consistency.py similarity index 69% rename from geobricks_data_scripts/prod/check_consistency/check_consistency.py rename to geobricks_data_scripts/prod/ghg/check_consistency/check_consistency.py index f927a9b..7377c6a 100644 --- a/geobricks_data_scripts/prod/check_consistency/check_consistency.py +++ b/geobricks_data_scripts/prod/ghg/check_consistency/check_consistency.py @@ -1,5 +1,5 @@ from geobricks_common.core.log import logger -from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.ghg.utils.data_manager_util import get_data_manager log = logger(__file__) diff --git a/geobricks_data_scripts/prod/config/.gitignore b/geobricks_data_scripts/prod/ghg/config/.gitignore similarity index 100% rename from geobricks_data_scripts/prod/config/.gitignore rename to geobricks_data_scripts/prod/ghg/config/.gitignore diff --git a/geobricks_data_scripts/demo/fenix/data/ghg/delete/__init__.py b/geobricks_data_scripts/prod/ghg/data/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/fenix/data/ghg/delete/__init__.py rename to geobricks_data_scripts/prod/ghg/data/__init__.py diff --git a/geobricks_data_scripts/demo/fenix/data/ghg/publish/__init__.py b/geobricks_data_scripts/prod/ghg/data/earthstat/__init__.py similarity index 100% rename from geobricks_data_scripts/demo/fenix/data/ghg/publish/__init__.py rename to geobricks_data_scripts/prod/ghg/data/earthstat/__init__.py diff --git a/geobricks_data_scripts/prod/check_consistency/__init__.py b/geobricks_data_scripts/prod/ghg/data/earthstat/delete/__init__.py similarity index 100% rename from geobricks_data_scripts/prod/check_consistency/__init__.py rename to geobricks_data_scripts/prod/ghg/data/earthstat/delete/__init__.py diff --git a/geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/prod/ghg/data/earthstat/delete/delete_all_earthstat.py similarity index 77% rename from geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py rename to geobricks_data_scripts/prod/ghg/data/earthstat/delete/delete_all_earthstat.py index 9571d55..f582983 100644 --- a/geobricks_data_scripts/prod/data/earthstat/delete/delete_all_earthstat.py +++ b/geobricks_data_scripts/prod/ghg/data/earthstat/delete/delete_all_earthstat.py @@ -1,4 +1,4 @@ -from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.ghg.utils.data_manager_util import get_data_manager data_manager = get_data_manager() layers = data_manager.get_all_layers() diff --git a/geobricks_data_scripts/prod/config/__init__.py b/geobricks_data_scripts/prod/ghg/data/earthstat/publish/__init__.py similarity index 100% rename from geobricks_data_scripts/prod/config/__init__.py rename to geobricks_data_scripts/prod/ghg/data/earthstat/publish/__init__.py diff --git a/geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py b/geobricks_data_scripts/prod/ghg/data/earthstat/publish/publish_harvest.py similarity index 84% rename from geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py rename to geobricks_data_scripts/prod/ghg/data/earthstat/publish/publish_harvest.py index 3758be7..786cda8 100644 --- a/geobricks_data_scripts/prod/data/earthstat/publish/publish_harvest.py +++ b/geobricks_data_scripts/prod/ghg/data/earthstat/publish/publish_harvest.py @@ -1,6 +1,6 @@ import glob import os -from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.ghg.utils.data_manager_util import get_data_manager from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder diff --git a/geobricks_data_scripts/prod/data/__init__.py b/geobricks_data_scripts/prod/ghg/data/ghg/__init__.py similarity index 100% rename from geobricks_data_scripts/prod/data/__init__.py rename to geobricks_data_scripts/prod/ghg/data/ghg/__init__.py diff --git a/geobricks_data_scripts/prod/data/ghg/createZipProducts.py b/geobricks_data_scripts/prod/ghg/data/ghg/createZipProducts.py similarity index 100% rename from geobricks_data_scripts/prod/data/ghg/createZipProducts.py rename to geobricks_data_scripts/prod/ghg/data/ghg/createZipProducts.py diff --git a/geobricks_data_scripts/prod/data/earthstat/__init__.py b/geobricks_data_scripts/prod/ghg/data/ghg/delete/__init__.py similarity index 100% rename from geobricks_data_scripts/prod/data/earthstat/__init__.py rename to geobricks_data_scripts/prod/ghg/data/ghg/delete/__init__.py diff --git a/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/prod/ghg/data/ghg/delete/data_manager_delete_all_example.py similarity index 81% rename from geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py rename to geobricks_data_scripts/prod/ghg/data/ghg/delete/data_manager_delete_all_example.py index 103ee07..b3e4123 100644 --- a/geobricks_data_scripts/prod/data/ghg/delete/data_manager_delete_all_example.py +++ b/geobricks_data_scripts/prod/ghg/data/ghg/delete/data_manager_delete_all_example.py @@ -1,4 +1,4 @@ -from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.ghg.utils.data_manager_util import get_data_manager data_manager = get_data_manager() layers = data_manager.get_all_layers() diff --git a/geobricks_data_scripts/prod/data/ghg/prepare_data.py b/geobricks_data_scripts/prod/ghg/data/ghg/prepare_data.py similarity index 86% rename from geobricks_data_scripts/prod/data/ghg/prepare_data.py rename to geobricks_data_scripts/prod/ghg/data/ghg/prepare_data.py index ef52c38..9319b57 100644 --- a/geobricks_data_scripts/prod/data/ghg/prepare_data.py +++ b/geobricks_data_scripts/prod/ghg/data/ghg/prepare_data.py @@ -1,5 +1,5 @@ -from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.ghg.utils.data_manager_util import get_data_manager # data_manager = get_data_manager() # layers = data_manager.get_all_layers() diff --git a/geobricks_data_scripts/prod/data/earthstat/delete/__init__.py b/geobricks_data_scripts/prod/ghg/data/ghg/publish/__init__.py similarity index 100% rename from geobricks_data_scripts/prod/data/earthstat/delete/__init__.py rename to geobricks_data_scripts/prod/ghg/data/ghg/publish/__init__.py diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data_old.py b/geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_GHG_data_old.py similarity index 96% rename from geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data_old.py rename to geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_GHG_data_old.py index f337628..e272701 100644 --- a/geobricks_data_scripts/prod/data/ghg/publish/publish_GHG_data_old.py +++ b/geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_GHG_data_old.py @@ -1,7 +1,7 @@ import os import glob from geobricks_common.core.log import logger -from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.ghg.utils.data_manager_util import get_data_manager from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder log = logger(__file__) diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py b/geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_data.py similarity index 98% rename from geobricks_data_scripts/prod/data/ghg/publish/publish_data.py rename to geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_data.py index 6fab038..75449e6 100644 --- a/geobricks_data_scripts/prod/data/ghg/publish/publish_data.py +++ b/geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_data.py @@ -1,7 +1,7 @@ import glob import os from geobricks_common.core.log import logger -from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.ghg.utils.data_manager_util import get_data_manager from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder log = logger(__file__) diff --git a/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py b/geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_sld.py similarity index 89% rename from geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py rename to geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_sld.py index 6683861..c0fdc63 100644 --- a/geobricks_data_scripts/prod/data/ghg/publish/publish_sld.py +++ b/geobricks_data_scripts/prod/ghg/data/ghg/publish/publish_sld.py @@ -1,4 +1,4 @@ -from geobricks_data_scripts.prod.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.prod.ghg.utils.data_manager_util import get_data_manager from geobricks_common.core.filesystem import get_filename import glob diff --git a/geobricks_data_scripts/prod/utils/.gitignore b/geobricks_data_scripts/prod/ghg/utils/.gitignore similarity index 100% rename from geobricks_data_scripts/prod/utils/.gitignore rename to geobricks_data_scripts/prod/ghg/utils/.gitignore diff --git a/geobricks_data_scripts/prod/data/earthstat/publish/__init__.py b/geobricks_data_scripts/prod/ghg/utils/__init__.py similarity index 100% rename from geobricks_data_scripts/prod/data/earthstat/publish/__init__.py rename to geobricks_data_scripts/prod/ghg/utils/__init__.py diff --git a/geobricks_data_scripts/prod/utils/data_manager_util.py b/geobricks_data_scripts/prod/ghg/utils/data_manager_util.py similarity index 66% rename from geobricks_data_scripts/prod/utils/data_manager_util.py rename to geobricks_data_scripts/prod/ghg/utils/data_manager_util.py index a3e9003..2efdd80 100644 --- a/geobricks_data_scripts/prod/utils/data_manager_util.py +++ b/geobricks_data_scripts/prod/ghg/utils/data_manager_util.py @@ -1,8 +1,8 @@ from geobricks_data_manager.core.data_manager_core import DataManager -from geobricks_data_scripts.prod.config.config import settings +from geobricks_data_scripts.prod.ghg.config.config import settings # get the default data manager for demo db def get_data_manager(): - print "Prod DAata Manager" + print "Prod Data Manager" print settings return DataManager(settings) \ No newline at end of file diff --git a/geobricks_data_scripts/prod/utils/__init__.py b/geobricks_data_scripts/prod/utils/__init__.py deleted file mode 100644 index 22d574d..0000000 --- a/geobricks_data_scripts/prod/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'vortex' From 9e71202824cb724ac2e6880455ff58b4b5b61796 Mon Sep 17 00:00:00 2001 From: simonefenix Date: Tue, 9 Jun 2015 11:23:44 +0200 Subject: [PATCH 36/39] refactoring --- .../earthstat/delete/delete_all_earthstat.py | 5 +- .../demo/fenix/data/ghg_NOT_USED/__init__.py | 1 + .../data/ghg_NOT_USED/delete/__init__.py | 1 + .../delete/data_manager_delete_all_example.py | 11 ++++ .../data/ghg_NOT_USED/publish/__init__.py | 1 + .../ghg_NOT_USED/publish/publish_GHG_data.py | 63 +++++++++++++++++++ 6 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/delete/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/delete/data_manager_delete_all_example.py create mode 100644 geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/publish/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/publish/publish_GHG_data.py diff --git a/geobricks_data_scripts/demo/fenix/data/earthstat/delete/delete_all_earthstat.py b/geobricks_data_scripts/demo/fenix/data/earthstat/delete/delete_all_earthstat.py index 6902f79..8345f21 100644 --- a/geobricks_data_scripts/demo/fenix/data/earthstat/delete/delete_all_earthstat.py +++ b/geobricks_data_scripts/demo/fenix/data/earthstat/delete/delete_all_earthstat.py @@ -4,8 +4,9 @@ layers = data_manager.get_all_layers() for layer in layers: try: - if "earthstat" in layer["dsd"]["workspace"]: - data_manager.delete(layer["uid"]) + print layer["dsd"] + # if "earthstat" in layer["dsd"]["workspace"]: + # data_manager.delete(layer["uid"]) except Exception, e: print "Error", e pass \ No newline at end of file diff --git a/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/__init__.py b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/delete/__init__.py b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/delete/data_manager_delete_all_example.py b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/delete/data_manager_delete_all_example.py new file mode 100644 index 0000000..6092615 --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/delete/data_manager_delete_all_example.py @@ -0,0 +1,11 @@ +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +for layer in layers: + try: + print layer["uid"] + data_manager.delete(layer["uid"]) + except Exception, e: + print e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/publish/__init__.py b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/publish/publish_GHG_data.py b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/publish/publish_GHG_data.py new file mode 100644 index 0000000..4075bef --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/ghg_NOT_USED/publish/publish_GHG_data.py @@ -0,0 +1,63 @@ +import os +import glob + +from geobricks_common.core.log import logger +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder + + +log = logger(__file__) + +data_manager = get_data_manager() + +workspace = "ghg" + + +def default_import(src_folder): + log.info("GLC2000: " + src_folder) + metadatas = harvest_folder(data_manager, src_folder, workspace, True, True, True) + + +def burned_areas(src_folder): + folders = glob.glob(os.path.join(src_folder, "*")) + for folder in folders: + folder = os.path.join(folder, "output") + log.info(folder) + metadatas = harvest_folder(data_manager, folder, workspace, True, True, True) + log.info(metadatas) + for metadata in metadatas: + log.info(metadata) + if "3857" in metadata["dsd"]["layerName"]: + log.info("here") + # log.info"metadata["dsd"]["datasource"]) + data_manager.geoserver_manager.set_style(metadata["dsd"]["layerName"], workspace, "burned_areas_EN") + + +# used just to set the style to modis that was wrong +def set_style_modis(): + data_manager.geoserver_manager.set_style("modis_land_cover_mod12q1_umd_500m_2009_3857", workspace, "modis_-_land_cover_type_umd_EN") + + +if __name__ == '__main__': + base_folder = "/home/vortex/Desktop/LAYERS/GHG_13_NOVEMEBRE/Theory_structure_novemeber_13/TO_PUBLISH" + + #Cultivation organic soils + default_import(os.path.join(base_folder, "Cultivation_organic_soils_croplands", "Cultivation_organic_soils_-_croplands")) + + # Burned areas + burned_areas(os.path.join(base_folder, "GFED4_BURNEDAREAS_BY_LANDCOVER")) + + # GLC 2000 (TODO: set the style) + #default_import(os.path.join(base_folder, "Global_Land_Cover_2000_(GLC_2000)", "Global_Land_Cover_2000_(GLC_2000)")) + + # griddedlivestock + default_import(os.path.join(base_folder, "GriddedLivestock", "gridded_livestock_of_the_world_v_201")) + + # JRC Climate zone + default_import(os.path.join(base_folder, "JRC_CLIMATE_ZONE", "JRC_climate_zone")) + + # MODIS_-_Land_Cover_type_UMD + default_import(os.path.join(base_folder, "MODIS_land_cover", "MODIS_-_Land_Cover_type_UMD")) + + + From c171516b3f01739c42a2bb49b655bb8ab3be96de Mon Sep 17 00:00:00 2001 From: simonefenix Date: Fri, 26 Jun 2015 17:39:50 +0200 Subject: [PATCH 37/39] update with eco --- .../demo/fenix/data/eco_countries/__init__.py | 1 + .../data/eco_countries/delete/__init__.py | 1 + .../delete/delete_all_eco_countries.py | 13 ++ .../data/eco_countries/publish/__init__.py | 1 + .../eco_countries/publish/publish_harvest.py | 120 ++++++++++++++++++ .../dev/data/eco_countries/__init__.py | 1 + .../dev/data/eco_countries/delete/__init__.py | 1 + .../delete/delete_all_eco_countries.py | 14 ++ .../data/eco_countries/publish/__init__.py | 1 + .../eco_countries/publish/publish_harvest.py | 14 ++ .../ghg/old/process/process_burned_areas.py | 2 +- .../utils/harvest/publish_harvest.py | 6 +- 12 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 geobricks_data_scripts/demo/fenix/data/eco_countries/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/eco_countries/delete/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/eco_countries/delete/delete_all_eco_countries.py create mode 100644 geobricks_data_scripts/demo/fenix/data/eco_countries/publish/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/eco_countries/publish/publish_harvest.py create mode 100644 geobricks_data_scripts/dev/data/eco_countries/__init__.py create mode 100644 geobricks_data_scripts/dev/data/eco_countries/delete/__init__.py create mode 100644 geobricks_data_scripts/dev/data/eco_countries/delete/delete_all_eco_countries.py create mode 100644 geobricks_data_scripts/dev/data/eco_countries/publish/__init__.py create mode 100644 geobricks_data_scripts/dev/data/eco_countries/publish/publish_harvest.py diff --git a/geobricks_data_scripts/demo/fenix/data/eco_countries/__init__.py b/geobricks_data_scripts/demo/fenix/data/eco_countries/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/eco_countries/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/eco_countries/delete/__init__.py b/geobricks_data_scripts/demo/fenix/data/eco_countries/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/eco_countries/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/eco_countries/delete/delete_all_eco_countries.py b/geobricks_data_scripts/demo/fenix/data/eco_countries/delete/delete_all_eco_countries.py new file mode 100644 index 0000000..3ba5b15 --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/eco_countries/delete/delete_all_eco_countries.py @@ -0,0 +1,13 @@ +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +print "layers" +for layer in layers: + try: + if "myd11c3_anomaly" in layer["dsd"]["workspace"]: + print layer["uid"] + #data_manager.delete(layer["uid"]) + except Exception, e: + #print "Error", e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/demo/fenix/data/eco_countries/publish/__init__.py b/geobricks_data_scripts/demo/fenix/data/eco_countries/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/eco_countries/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/eco_countries/publish/publish_harvest.py b/geobricks_data_scripts/demo/fenix/data/eco_countries/publish/publish_harvest.py new file mode 100644 index 0000000..700c024 --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/eco_countries/publish/publish_harvest.py @@ -0,0 +1,120 @@ +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style + + +data_manager = get_data_manager() + +lang = "EN" + +# ET +# workspace = "eco_et" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_et_avg" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_AVG", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_et_anomaly" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_ANOMALY", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_et_zscore" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_ZSCORE", workspace, True, False, False, None) +# print metadatas + +# workspace = "eco_et_anomaly_dpy" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_ANOMALY_DPY", workspace, True, False, False, None) + + + +# LST +# workspace = "eco_myd11c3" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_myd11c3_avg" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/MYD11C3_AVG", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_myd11c3_anomaly" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/MYD11C3_ANOMALY", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_myd11c3_zscore" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/MYD11C3_ZSCORE", workspace, True, False, False, None) +# print metadatas + +# workspace = "eco_myd11c3_anomaly_dpy" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/MYD11C3_ANOMALY_DPY", workspace, True, False, False, None) + + +# MOD13A3 +# workspace = "eco_mod13a3" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_mod13a3_avg" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/MOD13A3_AVG", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_mod13a3_anomaly" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/MOD13A3_ANOMALY", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_mod13a3_zscore" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/MOD13A3_ZSCORE/", workspace, True, False, False, None) + + +# workspace = "eco_mod13a3_anomaly_dpy" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/MOD13A3_ANOMALY_DPY", workspace, True, False, False, None) + +# RAINFED_LAND_GAEZ +# workspace = "wsp" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/RAINFED_LAND_GAEZ/", workspace, True, False, False, None) + +# Population LANDSCAN +# workspace = "wsp" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/POPULATION_LANDSCAN/", workspace, True, False, False, None) + +# Population LANDSCAN +# workspace = "wsp" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/POPULATION_LANDSCAN/", workspace, True, False, False, None) + +#CULTIVATED_LAND_GAEZ_2010 +#workspace = "wsp" +#metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CULTIVATED_LAND_GAEZ_2010/", workspace, True, False, False, None) + +#CULTIVATED_LAND_GAEZ_2010 +#workspace = "wsp" +#metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/IRRIGATED_AREAS_SOLAW_2012/", workspace, True, False, False, None) + + +# ECMWF +#workspace = "ecmwf" +#metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/ECMWF/", workspace, True, False, False, None) + +# CHIRPS +# workspace = "chirps" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/", workspace, True, False, False, None) + +# workspace = "chirps_avg" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/CHIRPS_AVG/", workspace, True, False, False, None) +# +# workspace = "chirps_anomaly" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/CHIRPS_ANOMALY/", workspace, True, False, False, None) +# +# workspace = "chirps_zscore" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/CHIRPS_ZSCORE//", workspace, True, False, False, None) +# +# workspace = "chirps_anomaly_dpy" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/CHIRPS_ANOMALY_DPY", workspace, True, False, False, None) + + +# HOTSPOT +# workspace = "eco" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/HOT_SPOT_ANALYSIS/DROUGHT_HOTPOST_AFG", workspace, True, False, False, None) + +# WHEAT ARE AFG +# workspace = "eco" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/HOT_SPOT_ANALYSIS/WHEAT_AREA_AFG", workspace, True, False, False, None) diff --git a/geobricks_data_scripts/dev/data/eco_countries/__init__.py b/geobricks_data_scripts/dev/data/eco_countries/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/eco_countries/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/eco_countries/delete/__init__.py b/geobricks_data_scripts/dev/data/eco_countries/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/eco_countries/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/eco_countries/delete/delete_all_eco_countries.py b/geobricks_data_scripts/dev/data/eco_countries/delete/delete_all_eco_countries.py new file mode 100644 index 0000000..6c699cb --- /dev/null +++ b/geobricks_data_scripts/dev/data/eco_countries/delete/delete_all_eco_countries.py @@ -0,0 +1,14 @@ + +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +print "layers" +for layer in layers: + try: + if "eco_countries" in layer["dsd"]["workspace"]: + data_manager.delete(layer["uid"]) + #print layer["uid"] + except Exception, e: + #print "Error", e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/eco_countries/publish/__init__.py b/geobricks_data_scripts/dev/data/eco_countries/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/dev/data/eco_countries/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/dev/data/eco_countries/publish/publish_harvest.py b/geobricks_data_scripts/dev/data/eco_countries/publish/publish_harvest.py new file mode 100644 index 0000000..5d64cdc --- /dev/null +++ b/geobricks_data_scripts/dev/data/eco_countries/publish/publish_harvest.py @@ -0,0 +1,14 @@ +import glob +import os +from geobricks_data_scripts.dev.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style + + +data_manager = get_data_manager() + +lang = "EN" +workspace = "eco_countries" + +metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_AVG/", workspace, True, False, False, None) +print metadatas +# harvest_folder(data_manager, "/home/vortex/Desktop/LAYERS/earthstat/earthstat_processeddata/geoserver/earthstat_crop_yield/", workspace) \ No newline at end of file diff --git a/geobricks_data_scripts/dev/data/ghg/old/process/process_burned_areas.py b/geobricks_data_scripts/dev/data/ghg/old/process/process_burned_areas.py index 2b71754..10f4bd0 100644 --- a/geobricks_data_scripts/dev/data/ghg/old/process/process_burned_areas.py +++ b/geobricks_data_scripts/dev/data/ghg/old/process/process_burned_areas.py @@ -89,7 +89,7 @@ def process(src_folder, dest_folder, file_type): process_steps[0]["source_path"] = [f] process_steps[0]["output_path"] = output_path process_steps[0]["output_file_name"] = output_file_name - for p in process_step: + for p in process_steps: p["source_path"] = p["source_path"] if "source_path" in p else result result = process_obj(p) elif prj == "4326": diff --git a/geobricks_data_scripts/utils/harvest/publish_harvest.py b/geobricks_data_scripts/utils/harvest/publish_harvest.py index 16f8774..9dcf09c 100644 --- a/geobricks_data_scripts/utils/harvest/publish_harvest.py +++ b/geobricks_data_scripts/utils/harvest/publish_harvest.py @@ -21,6 +21,8 @@ def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, p print metadata log.info(metadata) + + # TODO: rework the check on projection!! # check epsg if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] == "EPSG:3857": if "workspace" not in metadata["dsd"]: @@ -32,8 +34,8 @@ def harvest_folder(data_manager, folder, workspace, publish_on_geoserver=True, p # harvest 4326 (or others) to storage # it's done after the 3857 it's sure the metadata will be there for metadata in metadatas: - log.info(metadata) - if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:3857": + # log.info(metadata) + if metadata["meReferenceSystem"]["seProjection"]["projection"]["codes"][0]["code"] != "EPSG:4326": if publish_on_storage: metadata = publish_storage(data_manager, metadata["path"], metadata, workspace) log.info(metadata) From 244b71ff248e4cb045dfc22df0e12c97196ee6fc Mon Sep 17 00:00:00 2001 From: Simone Murzilli Date: Thu, 16 Jul 2015 09:43:08 +0200 Subject: [PATCH 38/39] Update README.md --- geobricks_data_scripts/README.md | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/geobricks_data_scripts/README.md b/geobricks_data_scripts/README.md index 499a99a..8b13789 100644 --- a/geobricks_data_scripts/README.md +++ b/geobricks_data_scripts/README.md @@ -1,27 +1 @@ -# Structure of geobricks_data_scripts -## geobricks_data_scripts.dev - -It's used on localhost (geoserver and d3s). - -**IMPORTANT**: It's used to **RENAME**, **MANAGE (i.e. move folders)** and **PROCESSING** the layers - -## geobricks_data_scripts.demo - -It's used for demo purposes and the data are **ALREADY BEEN PROCESSED**. It's used just to publish, delete and manage data. - -### geobricks_data_scripts.demo.fenix - -It uses: - -* D3S: fenix.fao.org/d3s -* GEOSERVER (master): lprapp16.fao.org:20200/geoserver/rest - - -### geobricks_data_scripts.demo.old_ghg - -**@Deprecated** - -## PROD - -**TODO** From f0921e61b6e75c1f02b0db321f7a810dbd24563d Mon Sep 17 00:00:00 2001 From: stefanocudini Date: Wed, 21 Oct 2015 11:06:33 +0200 Subject: [PATCH 39/39] added scripts for nena --- .../demo/fenix/data/nena_region/__init__.py | 1 + .../fenix/data/nena_region/delete/__init__.py | 1 + .../delete/delete_all_eco_countries.py | 14 ++ .../data/nena_region/publish/__init__.py | 1 + .../nena_region/publish/publish_harvest.py | 141 ++++++++++++++++++ 5 files changed, 158 insertions(+) create mode 100644 geobricks_data_scripts/demo/fenix/data/nena_region/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/nena_region/delete/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/nena_region/delete/delete_all_eco_countries.py create mode 100644 geobricks_data_scripts/demo/fenix/data/nena_region/publish/__init__.py create mode 100644 geobricks_data_scripts/demo/fenix/data/nena_region/publish/publish_harvest.py diff --git a/geobricks_data_scripts/demo/fenix/data/nena_region/__init__.py b/geobricks_data_scripts/demo/fenix/data/nena_region/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/nena_region/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/nena_region/delete/__init__.py b/geobricks_data_scripts/demo/fenix/data/nena_region/delete/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/nena_region/delete/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/nena_region/delete/delete_all_eco_countries.py b/geobricks_data_scripts/demo/fenix/data/nena_region/delete/delete_all_eco_countries.py new file mode 100644 index 0000000..cc64957 --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/nena_region/delete/delete_all_eco_countries.py @@ -0,0 +1,14 @@ +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager + + +data_manager = get_data_manager() +layers = data_manager.get_all_layers() +print "layers" +for layer in layers: + try: + if "nena_" in layer["dsd"]["workspace"]: + print layer["uid"] + data_manager.delete(layer["uid"]) + except Exception, e: + #print "Error", e + pass \ No newline at end of file diff --git a/geobricks_data_scripts/demo/fenix/data/nena_region/publish/__init__.py b/geobricks_data_scripts/demo/fenix/data/nena_region/publish/__init__.py new file mode 100644 index 0000000..22d574d --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/nena_region/publish/__init__.py @@ -0,0 +1 @@ +__author__ = 'vortex' diff --git a/geobricks_data_scripts/demo/fenix/data/nena_region/publish/publish_harvest.py b/geobricks_data_scripts/demo/fenix/data/nena_region/publish/publish_harvest.py new file mode 100644 index 0000000..b36245f --- /dev/null +++ b/geobricks_data_scripts/demo/fenix/data/nena_region/publish/publish_harvest.py @@ -0,0 +1,141 @@ +from geobricks_data_scripts.demo.fenix.utils.data_manager_util import get_data_manager +from geobricks_data_scripts.utils.harvest.publish_harvest import harvest_folder, update_folder_style +import getpass + +data_manager = get_data_manager() + +lang = "EN" + +# ET +# workspace = "eco_et" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_et_avg" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_AVG", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_et_anomaly" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_ANOMALY", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_et_zscore" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_ZSCORE", workspace, True, False, False, None) +# print metadatas + +# workspace = "eco_et_anomaly_dpy" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD16/ET/ET/ET_ANOMALY_DPY", workspace, True, False, False, None) + + + +# LST +# workspace = "eco_myd11c3" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_myd11c3_avg" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/MYD11C3_AVG", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_myd11c3_anomaly" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/MYD11C3_ANOMALY", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_myd11c3_zscore" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/MYD11C3_ZSCORE", workspace, True, False, False, None) +# print metadatas + +# workspace = "eco_myd11c3_anomaly_dpy" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MYD11C3/MYD11C3_ANOMALY_DPY", workspace, True, False, False, None) + + +# MOD13A3 +# workspace = "eco_mod13a3" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_mod13a3_avg" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/MOD13A3_AVG", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_mod13a3_anomaly" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/MOD13A3_ANOMALY", workspace, True, False, False, None) +# print metadatas +# +# workspace = "eco_mod13a3_zscore" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/MOD13A3_ZSCORE/", workspace, True, False, False, None) + + +# workspace = "eco_mod13a3_anomaly_dpy" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/MOD13A3/MOD13A3_ANOMALY_DPY", workspace, True, False, False, None) + +# RAINFED_LAND_GAEZ +# workspace = "wsp" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/RAINFED_LAND_GAEZ/", workspace, True, False, False, None) + +# Population LANDSCAN +# workspace = "wsp" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/POPULATION_LANDSCAN/", workspace, True, False, False, None) + +# Population LANDSCAN +# workspace = "wsp" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/POPULATION_LANDSCAN/", workspace, True, False, False, None) + +#CULTIVATED_LAND_GAEZ_2010 +#workspace = "wsp" +#metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CULTIVATED_LAND_GAEZ_2010/", workspace, True, False, False, None) + +#CULTIVATED_LAND_GAEZ_2010 +#workspace = "wsp" +#metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/IRRIGATED_AREAS_SOLAW_2012/", workspace, True, False, False, None) + + +# ECMWF +#workspace = "ecmwf" +#metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/ECMWF/", workspace, True, False, False, None) + +# CHIRPS +# workspace = "chirps" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/", workspace, True, False, False, None) + +# workspace = "chirps_avg" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/CHIRPS_AVG/", workspace, True, False, False, None) +# +# workspace = "chirps_anomaly" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/CHIRPS_ANOMALY/", workspace, True, False, False, None) +# +# workspace = "chirps_zscore" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/CHIRPS_ZSCORE//", workspace, True, False, False, None) +# +# workspace = "chirps_anomaly_dpy" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/CHIRPS/CHIRPS_ANOMALY_DPY", workspace, True, False, False, None) + + +# HOTSPOT +# workspace = "eco" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/HOT_SPOT_ANALYSIS/DROUGHT_HOTPOST_AFG", workspace, True, False, False, None) + +# WHEAT ARE AFG +# workspace = "eco" +# metadatas = harvest_folder(data_manager, "/media/vortex/LaCie/LaCie/ECO_COUNTRIES/HOT_SPOT_ANALYSIS/WHEAT_AREA_AFG", workspace, True, False, False, None) + +basedir = "NENA_MOD13A3" +baseworkspace = basedir.lower() +basepath = "/media/" + getpass.getuser() + "/LaCie/NENA_REGION/" + basedir + +# MOD13A3 NENA +workspace = baseworkspace +metadatas = harvest_folder(data_manager, basepath, workspace, True, False, False, None) +print metadatas + +workspace = baseworkspace + "_avg" +metadatas = harvest_folder(data_manager, basepath + basedir + "_AVG", workspace, True, False, False, None) +print metadatas + +workspace = baseworkspace + "_anomaly" +metadatas = harvest_folder(data_manager, basepath + "_ANOMALY", workspace, True, False, False, None) +print metadatas + +workspace = baseworkspace + "_zscore" +metadatas = harvest_folder(data_manager, basepath + "_ZSCORE", workspace, True, False, False, None) +print metadatas \ No newline at end of file