Skip to content
Snippets Groups Projects
ingestData.py 7.91 KiB
Newer Older
'''
Created on 21 MAY. 2019

@author: TKOSSOKO
'''
import os
import logging
import requests #sudo pip install requests
import json
import sys
import shutil #used to move a file
from pathlib import Path
from properties.p import Property
from typing import Iterable

logging.basicConfig(filename='ShareData.log', level=logging.DEBUG, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
# We get the url of the service we want to call
url_root = os.environ['BMAP_BACKEND_URL']

# get the metadata of a data and call the api
def ingestPrivateData(propertiesPath):   
    input_file = propertiesPath
    print('1- We start the ingestion process')
    print('2- The content of the file '+ input_file + ' is read')   
    
    allVariableAreFilled = True;
    if Path(input_file).is_file():
        # load properties file
        data_criteria = Property().load_property_files(input_file)
        # add the criteria from the properties file to the dictionary
        criteria_list = {}
        try :
            user_id = data_criteria['user_id']
            if len(user_id) > 0:
                criteria_list['userId'] =  user_id
                print(' 2.1- The user id is '+ user_id)
            else:
                allVariableAreFilled =  False;
                print(' 2.1- The user is mandatory')
        except:
            pass
        try : 
            data_path = data_criteria['data_path']
            if len(data_path) > 0:
                criteria_list['dataPath'] =  data_path 
                print(' 2.2- The data path is '+ data_path)
            else:
                allVariableAreFilled =  False;
                print(' 2.2- The data path is mandatory')
        except:
            pass
        try : 
edupin's avatar
edupin committed
            product_type = data_criteria['product_type']
            if len(product_type) > 0:
                criteria_list['dataFormat'] =  product_type 
                print(' 2.3- The product type is '+ product_type)
            else:
                allVariableAreFilled =  False;
edupin's avatar
edupin committed
                print(' 2.3- The product type is mandatory')
        except:
            pass        
               
        # location = data_criteria['location']
        try :
            polarization = data_criteria['polarization']
            if len(polarization) > 0:
                criteria_list['polarization'] =  polarization 
                print(' 2.4- The polarization is '+ polarization)
            else:
edupin's avatar
edupin committed
                criteria_list['polarization'] =  None 
            '''
                allVariableAreFilled =  False;
                print(' 2.4- The polarization is mandatory')
edupin's avatar
edupin committed
            '''
        except:
            pass


        try : 
            sub_region_name = data_criteria['sub_region_name']
            if len(sub_region_name) > 0:
                criteria_list['subregionName'] =  sub_region_name 
                print(' 2.5- The sub region is '+ sub_region_name)
edupin's avatar
edupin committed
                '''
                allVariableAreFilled =  False;
edupin's avatar
edupin committed
                print(' 2.5- The sub region is mandatory')
edupin's avatar
edupin committed
                '''
                criteria_list['subregionName'] =  None 
                       
        except:
            pass
edupin's avatar
edupin committed
        '''
        try : 
            collection_name = data_criteria['collection_name']
            if len(collection_name) > 0:
                criteria_list['collectionName'] =  collection_name 
                print(' 2.4- The collection is '+ collection_name)
            else:
                allVariableAreFilled =  False;
                print(' 2.4- The collection is mandatory')     
        except:
            pass
edupin's avatar
edupin committed
        '''
            

    else:
        print('ERROR: The file "' + input_file + '" does not exist.')
        logging.error('The file "' + input_file + '" does not exist.')   
        
    if(allVariableAreFilled):
        print("All variables in sharedata.properties are filled. We start the process")
        ##We move the data in the /data/private using the user id
        print("3- Starting the copy of the file to a temp folder")
        folder_destination="/app/User_data/"
        
        folder_destination = createIfnotExistFolder(folder_destination, criteria_list['userId'])
        
        if(criteria_list['dataFormat'])== 'ROI' :
                print("3.1- File format detected is ROI. We need 4 files : dbf, shp, shx, prj") 
                listeOfRoi = isROIfileExistAll(data_path)   
                if(len(listeOfRoi)>0):
                    #We move the 4 ROI files                    
                    for i in range(len(listeOfRoi)):
                        filePath = listeOfRoi[i]        
                        shutil.copy(filePath, folder_destination)  
                    print("4- Files correctly copied")
                else:
                    print("4-we cannot ingest this ROI files. Some files are missing")
                    exit()
        else :
                shutil.copy(data_path, folder_destination)
                print("4- File correctly copied")
                
        ######We call the back end service to start the ingestion
        ingestTheData(criteria_list, folder_destination+"/"+os.path.basename(criteria_list['dataPath']))
edupin's avatar
edupin committed
        print("Not all mandatory fields are filled.")
        
        
#########Function to call the back end and check the id of the user      
def createIfnotExistFolder(directory, user_id):
    response = sendARequest(url_root+"bmapuser/"+user_id)
    if len(response) > 0:
        bmaap_user = json.loads(response)
        id = bmaap_user["BmaapUser"]["id"]
        pathToTempData = directory+str(id)
        if not os.path.exists(pathToTempData):
           os.makedirs(pathToTempData)
        return pathToTempData
    else:
        print("The user is unknown. We stop the process")




#########Function to send a request to the back end
def sendARequest(url) :
    try:
        response = requests.get(url)
        json_str = response.text
        # if the response body contains something
        if len(json_str) > 0:
            return json_str
        else:
            return ""
    except requests.exceptions.RequestException as e:
            print('ERROR: ' + str(e))
            logging.error(str(e))
#########################################################################
edupin's avatar
edupin committed
def ingestTheData(criteria_list, destination):
       
        print('5- Your metadata are : ' + str(criteria_list))
edupin's avatar
edupin committed
        print('destination')
        url = url_root + "catalogue/granule/private/add?dataPath=" + destination  + "&dataFormat=" + criteria_list['dataFormat'] + "&userId=" + str(criteria_list['userId'])
        print(url)
edupin's avatar
edupin committed
		if criteria_list['subregionName']
			url = url+ "&subregionName=" + criteria_list['subregionName']
		if criteria_list['polarization']
			url = url+ "&polarization=" + criteria_list['polarization']
		
		print(url)
edupin's avatar
edupin committed
        #"&subregionName=" + criteria_list['subregionName'] + 
        #+ "&polarization=" +criteria_list['polarization']
        response = requests.get(url)
        json_str = response.text
        code = response.status_code
        if (code == 200):
            print("File correctly ingested")
        else:
            print(json_str)
######################################

def isROIfileExistAll(datapath) :
    dataPathWithouExtension =(os.path.splitext(datapath)[0])
    roiShx = dataPathWithouExtension + ".shx"
    roiShp = dataPathWithouExtension + ".shp"
    roiDbf = dataPathWithouExtension + ".dbf"
    roiPrj = dataPathWithouExtension + ".prj"
    #We start the verification of the extension
    listeOfRoi = []
    listeOfRoi.extend((roiShx, roiShp, roiDbf, roiPrj))
    for i in range(len(listeOfRoi)):
        filePath = listeOfRoi[i]
        if Path(filePath).is_file():
edupin's avatar
edupin committed
            print("ROI file found at  "+filePath)
edupin's avatar
edupin committed
            print("A mandatory ROI file is missing "+filePath)
            listeEmpty = []
            return listeEmpty
    
    return listeOfRoi
    
######################################

ingestPrivateData(sys.argv[1])