added COPA light stack Dockerfile

Dockerfile

@@ -38,7 +38,8 @@ RUN  apt-get update && \
       rm -rf geopandas && \
       apt-get -y install libgdal1-dev && \
       pip install --user fiona && \
-       apt-get install python-shapely 
+       apt-get install python-shapely && \
+       pip install opencv-python
      
 #INstallation of Octave
 RUN   apt-add-repository ppa:octave/stable && \

python_stack_image/Dockerfile

+#   Capgemini for Biomass project
+#   Author CBall
+#   Date first created 27/03/2019
+FROM ubuntu:16.04
+
+LABEL maintainer="caroline.ball@capgemini.com"
+LABEL version="1.0"
+LABEL description="Docker file based on ubuntu for python algorithms for Biomass project"
+
+#INSTALL GDAL
+RUN apt-get update &&\
+    echo "deb http://ppa.launchpad.net/ubuntugis/ppa/ubuntu xenial main" >> /etc/apt/sources.list &&\
+    echo "deb-src http://ppa.launchpad.net/ubuntugis/ppa/ubuntu xenial main" >> /etc/apt/sources.list &&\
+    apt-get install -y software-properties-common python-software-properties &&\
+    add-apt-repository ppa:ubuntugis/ppa &&\
+    apt-get update &&\
+    apt-get install -y gdal-bin python3-gdal 
+
+#UPGRADE PIP
+RUN apt-get update && apt-get install -y --reinstall python3-pip && \
+        apt-get -y install libsm6 libxext6 libxrender-dev && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+
+#ADD SCRIPT FILES
+COPY ./scripts /usr/bmap/
+COPY ./requirements.txt /usr/bmap/requirements.txt
+
+#INSTALL REQUIRES LIBRAIRIES 
+#RUN apt-get install -y python3-lxml
+RUN pip3 install --upgrade pip && \
+    pip --no-cache-dir install -r /usr/bmap/requirements.txt
+
+# Add Librairies for RestClient.py
+RUN pip install requests
+
+ENV PATH="/usr/bmap/:${PATH}"
+ENV PYTHONPATH="/usr/bmap/:${PYTHONPATH}"
+ENV BMAP_BACKEND_URL=http://backend-val.biomass-maap.com/bmap-web/
+
+# TEST IMPORT GDAL
+CMD ["python3"]

python_stack_image/build.sh

+docker build -t kosted/bmap-stack-python:1.0 . 
+docker run --rm kosted/bmap-stack-python:1.0 pip freeze
+docker push kosted/bmap-stack-python:1.0 

python_stack_image/requirements.txt

+attrs==19.1.0
+autopep8==1.3.5
+certifi==2019.3.9
+chardet==3.0.4
+Click==7.0
+click-plugins==1.1.1
+cligj==0.5.0
+conda==4.3.16
+cycler==0.10.0
+Fiona==1.8.6
+future==0.16.0
+GDAL==2.2.2
+geopandas==0.5.0
+idna==2.8
+jedi==0.12.1
+joblib==0.13.2
+kiwisolver==1.1.0
+matplotlib==3.0.3
+mccabe==0.6.1
+munch==2.3.2
+numpy==1.16.3
+pandas==0.24.2
+parso==0.3.0
+Pillow==6.0.0
+pluggy==0.6.0
+property==2.2
+pycodestyle==2.4.0
+pycosat==0.6.3
+pydocstyle==2.1.1
+pyflakes==2.0.0
+pyparsing==2.4.0
+pyproj==2.1.3
+python-dateutil==2.8.0
+configparser==3.7.4
+futures==3.1.1
+python-language-server==0.19.0
+pytz==2019.1
+requests==2.22.0
+rope==0.10.7
+ruamel.yaml==0.15.96
+scikit-learn==0.21.2
+scipy==0.19.1
+Shapely==1.6.4.post2
+six==1.11.0
+snowballstemmer==1.2.1
+urllib3==1.25.3
+virtualenv==16.0.0
+yapf==0.22.0
+opencv-python==4.1.0.25

python_stack_image/scripts/RestClient.py

+'''
+Created on 13 nov. 2018
+
+@author: QFAURE
+'''
+import os
+import logging
+import requests #sudo pip install requests
+import json
+from pathlib import Path
+from properties.p import Property
+from typing import Iterable
+
+
+
+logging.basicConfig(filename='RestClient.log', level=logging.DEBUG, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+url = os.environ['BMAP_BACKEND_URL'] + 'catalogue/granule/'
+
+# gets a granule by name and returns the json containing the granule's metadata.
+def get_granule_by_name(granule_name: str,formatmetadata=False) -> Iterable:
+    try:
+        response = requests.get(url + 'granulename/' + granule_name)
+        json_str = response.text
+      
+        # if the response body contains something
+        if len(json_str) > 0:
+            json_obj = json.loads(json_str)
+            
+            if formatmetadata==True :
+                            
+                return format_metadata([json_obj])
+            else : 
+                return json_obj
+        else:
+            print('INFO: There is no data with ID: ' + granule_name)
+            logging.info('There is no data with ID: ' + granule_name)
+            return {}
+    except requests.exceptions.RequestException as e:
+        print('ERROR: ' + str(e))
+        logging.error(str(e))
+
+
+# gets granules by criteria and returns json containing granules' metadata:
+def get_granules_by_criteria(input_file: str='datacriteria.properties',formatmetadata=False) -> Iterable:
+    
+    
+    if Path(input_file).is_file():
+        # load properties file
+        data_criteria = Property().load_property_files(input_file)
+        # add the criteria from the properties file to the dictionary
+        criteria_list = {}
+        try :
+            start_date = data_criteria['start_date']
+            if len(start_date) > 0:
+                criteria_list['"startDate"'] = '"' + start_date + '"'
+        except:
+            pass
+        try : 
+            end_date = data_criteria['end_date']
+            if len(end_date) > 0:
+                criteria_list['"endDate"'] = '"' + end_date + '"'
+        except:
+            pass
+        try :    
+            product_types = data_criteria['product_types']
+            if len(product_types) > 0:
+                criteria_list['"productTypes"'] = to_table_string(product_types)
+        except:
+            pass
+        try :    
+            instrument_names = data_criteria['instrument_names']
+            if len(instrument_names) > 0:
+                criteria_list['"instrumentNames"'] = to_table_string(instrument_names)
+        except:
+            pass
+        
+        # location = data_criteria['location']
+        try :
+            polarizations = data_criteria['polarizations']
+            if len(polarizations) > 0:
+                criteria_list['"polarizations"'] = to_table_string(polarizations)
+        except:
+            pass
+        try : 
+            geometry_types = data_criteria['geometry_types']
+            if len(geometry_types) > 0:
+                criteria_list['"geometryTypes"'] = to_table_string(geometry_types)
+        except:
+            pass
+        try : 
+            processing_levels = data_criteria['processing_levels']
+            if len(processing_levels) > 0:
+                criteria_list['"processingLevels"'] = to_table_string(processing_levels)
+        except:
+            pass
+        try : 
+            sub_region_names = data_criteria['sub_region_names']
+            if len(sub_region_names) > 0:
+                criteria_list['"subRegionNames"'] = to_table_string(sub_region_names) 
+        except:
+            pass
+        try : 
+            collectionNames = data_criteria['collection_names']
+            if len(collectionNames) > 0:
+                criteria_list['"collectionNames"'] = to_table_string(collectionNames)
+        except:
+            pass
+        # if there is at least one criterion besides scene
+        if len(criteria_list) >= 1:
+            # concatenate the criteria from the dictionary together in a string (i.e. "key1": "value1", "key2: "value2")
+            criteria_str = ""
+            for i in criteria_list:
+                key = i
+                value = criteria_list[i]
+                if len(value) > 0:
+                    if len(criteria_str) > 0:
+                        criteria_str += ', '
+                    criteria_str += key + ': ' + value
+            # add the concatenated criteria string it inside a string representing a data criteria json
+            request_body = '{"GranuleCriteria": {' + criteria_str + '}}'
+            try:
+                response = requests.post(url, headers={'content-type': 'application/json'}, data=request_body)
+                json_str = response.text
+                # if the response body contains something
+                if len(json_str) > 0:
+                    json_obj = json.loads(json_str)
+                    #return filter_by_scene(json_obj, data_criteria['scene_name'].split(','))
+                    if data_criteria['scene_name']:
+                        json_obj=filter_by_scene(json_obj, data_criteria['scene_name'].split(','))
+                        if formatmetadata == True:
+                            return format_metadata(json_obj)
+                        else:
+                            return json_obj
+                    else:
+                        if formatmetadata==True:
+                            return format_metadata(json_obj)
+                        else : 
+                            return json_obj
+                else:
+                    print('INFO: There is no data matching the given criteria.')
+                    logging.info('There is no data matching the given criteria.')
+                    return {}
+            except requests.exceptions.RequestException as e:
+                print('ERROR: ' + str(e))
+                logging.error(str(e))
+        else:
+            print('ERROR: You need to specify at least one search criteria besides scene name.')
+            logging.error('You need to specify at least one search criteria besides scene name.')
+    else:
+        print('ERROR: The file "' + input_file + '" does not exist.')
+        logging.error('The file "' + input_file + '" does not exist.')
+
+
+# returns the string passed as parameter as a string representing a table (e.g. ["a", "b", "c"])
+def to_table_string(string: str) -> str:
+    string_to_return = string.replace(', ', ',').replace(',', '", "')
+    return '["' + string_to_return + '"]'
+
+
+def filter_by_scene(json_result, scene_list):
+    for granule in json_result[:] : 
+        scene = granule['Granule']['granuleScene']
+        if scene:
+            scene=granule['Granule']['granuleScene']['Granule']['name']
+            if not scene in scene_list : 
+                json_result.remove(granule)
+             
+    return json_result
+
+# download granule data(s) in a specified dir:
+# granule_id: granuleID.
+# target_Dir: directory the file(s) will be saved at".
+def download_granule(granule_id: str, target_Dir: str) -> Iterable:
+    try:
+        response = requests.get(url + 'granulename/' + granule_id)
+        json_str = response.text
+        print(target_Dir)
+        # if the response body contains something
+        if len(json_str) > 0:
+            json_obj = json.loads(response.text)
+            for data in json_obj['Granule']['dataList'][:] :
+                urlToData = data['Data']['urlToData']
+                completeName = os.path.join(target_Dir, data['Data']['fileName'])
+                r = requests.get(urlToData, allow_redirects=True)
+                open(completeName, 'wb').write(r.content)
+            return urlToData
+        else:
+            print('INFO: There is no data with ID: ' + granule_id)
+            logging.info('There is no data with ID: ' + granule_id)
+            return {}
+    except requests.exceptions.RequestException as e:
+        print('ERROR: ' + str(e))
+        logging.error(str(e))
+
+
+
+
+#########################################################################
+
+# -*- coding: utf-8 -*-
+class dataset:
+    def __init__(self):
+        """
+        This class contains the dataset and all the auxiliary information"""
+        self.campaign = ""
+        self.scenes = []
+        self.SLClist = {}
+        self.SLCFilenames = {}
+        self.heading={}
+        self.z_flight={}
+        self.z_terrain={}
+        self.GRD_resol={}
+        self.pixel_spacing = {}
+        self.surface_resol = {}
+        self.SLR_start = {}
+        self.master = {}
+        self.demname = {}
+        self.demFilenames= {}
+        self.incFilenames = {}
+        self.kzFilenames = {}
+        self.azimuthFiles = {}
+        self.rangeFiles = {}
+        
+def get_url(collection,granulename):
+    granule=get_granule_by_name(collection+':@'+granulename)
+    datalist=granule['Granule']['dataList']
+    
+    url=datalist[0]['Data']['filePath']
+    return(url)
+
+def format_metadata(json_result):
+    """Parse configuration xml and init dataset class"""
+    import re
+    
+    
+    # Init
+    data_stack = dataset()
+    
+    data_stack.campaign = json_result[0]['Granule']['collection']['Collection']['shortName']
+    #if the research asked some specific scenes
+    
+    
+        
+    # All the data of the collections are returned and the scene list must be extract from the granules
+    for granule in json_result :
+        if granule['Granule']['productType']=='SLC':
+            scene=granule['Granule']['granuleScene']['Granule']['name']   
+            #add scene in the list avoiding duplicates
+            
+            
+            ## inputfilenames
+            polarisation=granule['Granule']['polarization']
+            key=(scene,polarisation)
+            value=granule['Granule']['name']
+            url=get_url(data_stack.campaign,value)
+            
+            data_stack.SLClist.update(dict([(key,value)]))
+            data_stack.SLCFilenames.update(dict([(key,url)]))
+            if not scene in data_stack.scenes:
+                #append scenes and all metadata of the scene
+                data_stack.scenes.append(scene)            
+                
+                
+                    
+                #### heading
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['heading'])])
+                data_stack.heading.update(keyval)
+                #z_flight
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['zFlight'])])
+                data_stack.z_flight.update(keyval)
+                ##z_terrain
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['zTerrain'])])
+                data_stack.z_terrain.update(keyval)
+                ##GRD_resol
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['grdResol'])])
+                data_stack.GRD_resol.update(keyval)
+                ##pixel_spacing
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['pixelSpacing'])])
+                data_stack.pixel_spacing.update(keyval)
+                ##surface_resol
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['surfaceResol'])])
+                data_stack.surface_resol.update(keyval)
+                
+                ##SLR_start
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['slrStart'])])
+                data_stack.SLR_start.update(keyval)
+                
+                ##master
+                #search for the presence of az files in the file list of the granule
+                #bool_list=bool(re.search("az.tiff",data)) for data in granule['Granule']['granuleScene']['Granule']['granuleList']]                    
+                #if True in bool_list : 
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['master'])])
+                data_stack.master.update(keyval)
+                ##demname
+                keyval=dict([(scene,granule['Granule']['granuleScene']['Granule']['dem'])])
+                data_stack.demname.update(keyval)
+                
+                ##demFilenames
+                demname=granule['Granule']['granuleScene']['Granule']['dem']
+                #if a dem is referenced, try to get its filename
+                if len(demname)>0:
+                    try :
+                        demfilename=get_url(data_stack.campaign,data_stack.campaign+'_'+demname+'_dem.tiff')
+                        
+                    except:
+                        demfilename=''
+                else : 
+                        demfilename=''
+                keyval=dict([(scene,demfilename)])
+                data_stack.demFilenames.update(keyval)
+                
+                ##azfilenames
+                #search for the presence of az files in the file list of the granule
+                bool_list=[bool(re.search("az.tiff",data)) for data in granule['Granule']['granuleScene']['Granule']['granuleList']]     
+                if True in bool_list :
+                    azfile=granule['Granule']['granuleScene']['Granule']['granuleList'][bool_list.index(True)]
+                    azfile=get_url(data_stack.campaign,azfile)
+                elif  granule['Granule']['granuleScene']['Granule']['master']!='n/a':
+                    #get master name and get its azimtuh
+                    master=granule['Granule']['granuleScene']['Granule']['master']
+                    
+                    if len(master)>0:
+                        try : 
+                            master_granules=get_granule_by_name(data_stack.campaign+':@'+master)['Granule']['granuleList']
+                            
+                            azfile=[file for file in master_granules if "az" in file][0]      
+                            azfile=get_url(data_stack.campaign,azfile)
+                        except: 
+                            azfile=''
+                    else:
+                        azfile=''
+                else:
+                    azfile=''
+                keyval=dict([(scene,azfile)])
+                data_stack.azimuthFiles.update(keyval)
+                
+                ##rgfilenames
+                bool_list=[bool(re.search("rg.tiff",data)) for data in granule['Granule']['granuleScene']['Granule']['granuleList']]     
+                if True in bool_list :
+                    rgfile=granule['Granule']['granuleScene']['Granule']['granuleList'][bool_list.index(True)]
+                    rgfile=get_url(data_stack.campaign,rgfile)
+                elif  granule['Granule']['granuleScene']['Granule']['master']!='n/a':
+                    #get master name and get its azimtuh
+                    master=granule['Granule']['granuleScene']['Granule']['master']
+                    if len(master)>0:
+                        try : 
+                            master_granules=get_granule_by_name(data_stack.campaign+':@'+master)['Granule']['granuleList']
+                            rgfile=[file for file in master_granules if "rg" in file][0]    
+                            rgfile=get_url(data_stack.campaign,rgfile)
+                        except: 
+                            rgfile=''
+                    else:
+                        rgfile=''
+                else:
+                    rgfile=''
+                keyval=dict([(scene,rgfile)])
+                data_stack.rangeFiles.update(keyval)
+                
+                ##incfilenames
+                bool_list=[bool(re.search("inc.tiff",data)) for data in granule['Granule']['granuleScene']['Granule']['granuleList']]     
+                if True in bool_list :
+                    incfile=granule['Granule']['granuleScene']['Granule']['granuleList'][bool_list.index(True)]
+                    incfile=get_url(data_stack.campaign,incfile)
+                elif  granule['Granule']['granuleScene']['Granule']['master']!='n/a':
+                    #get master name and get its azimtuh
+                    master=granule['Granule']['granuleScene']['Granule']['master']
+                    if len(master)>0:
+                        try : 
+                            master_granules=get_granule_by_name(data_stack.campaign+':@'+master)['Granule']['granuleList']
+                            incfile=[file for file in master_granules if "inc" in file][0] 
+                            incfile=get_url(data_stack.campaign,incfile)
+                        except: 
+                            incfile=''
+                    else:
+                        incfile=''
+                else:
+                    incfile=''
+                keyval=dict([(scene,incfile)])
+                data_stack.incFilenames.update(keyval)
+                               
+                ##kzfilenames
+                bool_list=[bool(re.search("kz.tiff",data)) for data in granule['Granule']['granuleScene']['Granule']['granuleList']]     
+                if True in bool_list :
+                    kzfile=granule['Granule']['granuleScene']['Granule']['granuleList'][bool_list.index(True)]
+                    kzfile=get_url(data_stack.campaign,kzfile)
+                else:
+                    kzfile=''
+                keyval=dict([(scene,kzfile)])
+                data_stack.kzFilenames.update(keyval)
+    return data_stack
+    

python_stack_image/scripts/quicklook_raster.py

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Dec  5 23:55:39 2018
+
+@author: plousser
+
+"""
+import os, os.path, optparse, sys
+import gdal
+
+def quickL(inputFilename, quickLookBaseFilename, GRD_resol, Type, quickLookExtension):
+    'Create and save a quick look of the input image'
+    import os
+    from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
+    from matplotlib.figure import Figure
+    from matplotlib.colors import LinearSegmentedColormap
+    import numpy as np
+    
+    # Open original image in slant range geometry:
+    input_image_driver = gdal.Open(inputFilename, 0)
+    
+    ratio = max(1, input_image_driver.RasterXSize // 1000, input_image_driver.RasterYSize // 1000) # Ratio of the reduction of the read image (to avoid memory errors) 
+    input_image = input_image_driver.ReadAsArray(0, 0, None, None, None, input_image_driver.RasterXSize//ratio, input_image_driver.RasterYSize//ratio)
+    
+    # Define figure:
+    fig = Figure()
+    canvas = FigureCanvas(fig)
+    ax = fig.add_subplot(111)
+    
+    # Define custom color map:
+    if Type == 'biomass':
+        customCmap = LinearSegmentedColormap.from_list('biomass', ['#F5F5DC', 'g', '#004500'])
+    if Type == 'height':
+#        customCmap = LinearSegmentedColormap.from_list('height', ['#F5F5DC', '#966F33', '#452400'])
+        customCmap = 'jet'
+    if Type== 'sig':
+        customCmap='gray'
+        
+    # Display image:
+    if Type =='sig':
+        cax = ax.imshow(input_image, cmap=customCmap, vmin=0,vmax=np.nanpercentile(input_image,90),extent=(0, input_image_driver.RasterXSize*GRD_resol, 0, input_image_driver.RasterYSize*GRD_resol))
+    else : 
+        cax = ax.imshow(input_image, cmap=customCmap, extent=(0, input_image_driver.RasterXSize*GRD_resol, 0, input_image_driver.RasterYSize*GRD_resol))
+   
+    # Set figure Layout:
+    if len(gdal.Open(inputFilename).GetProjection())>0 :
+        ax.set_xlabel('Longitude (m)')
+        ax.set_ylabel('Latitude (m)')
+    else :
+        ax.set_xlabel('')
+        ax.set_ylabel('')
+    cbar = fig.colorbar(cax)
+    if Type == 'biomass':
+        cbar.set_label('Biomass (ton/ha)')
+    if Type == 'height':
+        cbar.set_label('Height (m)')
+    if Type== 'sig':
+        cbar.set_label('Backscattering')
+    ax.set_title(os.path.basename(quickLookBaseFilename) + '.tiff\n')
+    
+    # Save figure:
+    canvas.print_figure(quickLookBaseFilename + '_' + Type + '_QL.' + quickLookExtension, dpi=150, bbox_inches='tight')
+    
+    # Close image data set:
+    input_image_driver = None
+    
+###########################################################################
+class OptionParser (optparse.OptionParser):
+ 
+    def check_required (self, opt):
+        option = self.get_option(opt)
+ 
+        # Assumes the option's 'default' is set to None!
+        if getattr(self.values, option.dest) is None:
+            self.error("%s option not supplied" % option)
+ 
+               
+
+##########################################################################                
+            
+if __name__ == '__main__':
+    
+      #==================
+    #parse command line
+    #==================
+    if len(sys.argv) == 1:
+        prog = os.path.basename(sys.argv[0])
+        print ("      "+sys.argv[0]+' [options]')
+        print ("     Aide : ", prog, " --help")
+        print ("        ou : ", prog, " -h")
+        print ("example 1 : python %s -i inputdir/ -o outputdir/"%sys.argv[0])
+        
+        sys.exit(-1)
+    else :
+        usage = "usage: %prog [options] "
+    parser = OptionParser(usage=usage)
+  
+
+    parser.add_option("-i", "--inputfile", dest="inputfile", action="store", type="string", \
+            help="Path to the input file",default='.')
+    parser.add_option("-g", "--grdresol", dest="grdresol", action="store", type="float", \
+            help="GRD resol",default=2)
+    parser.add_option("-q", "--qlextension", dest="qlextension", action="store", type="choice", \
+            help="Output extension format",choices=['png','jpg','eps'],default='png')
+    parser.add_option("-t", '--type', dest="type", action="store", type="choice", \
+            help="Quicklook type",choices=['biomass','height','sig'],default='png')
+    
+    (options, args) = parser.parse_args()
+
+    quickL(options.inputfile, options.inputfile.replace('.tiff','quicklook'), options.grdresol, options.type, options.qlextension) 
+