Skip to content
Snippets Groups Projects
Commit 07825da2 authored by Herman's avatar Herman
Browse files

BIOMASS-2784 : update url token in notebook

parent 00452a46
No related branches found
No related tags found
1 merge request!13BIOMASS-2784 : update url token in notebook
Hide whitespace changes
Inline Side-by-side
init.sh
+ 1
− 1
View file @ 07825da2
......@@ -13,7 +13,7 @@ echo "user_id = $3" >> $HOME/.maap/auth.ini
touch $HOME/.maap/maap.ini
echo "[maap]" > $HOME/.maap/maap.ini
echo "copa_backend_url = https://gravitee-gateway.${MAAP_ENV_TYPE,,}.esa-maap.org/" >> $HOME/.maap/maap.ini
echo "url_token = https://iam.${MAAP_ENV_TYPE,,}.esa-maap.org/oxauth/restv1/token" >> $HOME/.maap/maap.ini
echo "url_token = https://auth.${MAAP_ENV_TYPE,,}.esa-maap.org/${$REALM_NAME}/maap/protocol/openid-connect/token" >> $HOME/.maap/maap.ini
echo "client_id = $CLIENT_ID" >> $HOME/.maap/maap.ini
echo "url_gravitee_s3 = https://gravitee-gateway.${MAAP_ENV_TYPE,,}.esa-maap.org/s3/" >> $HOME/.maap/maap.ini
......
scalability-s2-jupyter/Notebook_Classification_S2.ipynb
+ 1
− 1
View file @ 07825da2
%% Cell type:markdown id: tags:
# <center>Classification Sentinel 2 using wps</center>
%% Cell type:markdown id: tags:
1) Import Librairies
%% Cell type:code id: tags:
``` python
import requests
import json
import sys
import os
import xml.etree.ElementTree as ET
import time
import sys
sys.path.insert(0, '/projects/demo-scripts/common/')
from owslib.wps import WebProcessingService
import configparser
```
%% Cell type:code id: tags:
``` python
#Load the file tha contains maap values
config = configparser.ConfigParser()
config.read('/projects/.maap/maap.ini')
#Retrieve maap values
copa_backend_url = config['maap']['copa_backend_url']
CLIENT_ID = config['maap']['client_id']
url_token = config['maap']['url_token']
```
%% Cell type:code id: tags:
``` python
#Load the file tha contains auth values
config = configparser.ConfigParser()
config.read('/projects/.maap/auth.ini')
#Location of the credentials
email = config['auth']['email']
password = config['auth']['password']
user_id = config['auth']['user_id']
```
%% Cell type:code id: tags:
``` python
#retrieve a token
response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "openid+profile"})
response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "profile"})
#Convert the string to json to fecth access_token
data = json.loads(response.text)
token = data['access_token']
oauth_token = token
```
%% Cell type:code id: tags:
``` python
wps = WebProcessingService(copa_backend_url + 'wps/capabilities', \
headers = {'Authorization': 'Bearer ' + oauth_token}, version ="2.0.0")
print("--- GET CAPABILITIES---")
print("--- Service identification ---")
print('title: ',wps.identification.title)
print('type: ',wps.identification.type, wps.identification.version)
```
%% Cell type:code id: tags:
``` python
print("--- Service Provider ---")
print('provider: ',wps.provider.name,wps.provider.contact.email,wps.provider.url)
```
%% Cell type:code id: tags:
``` python
print("--- Service Operations ---")
for operation in wps.operations:
print(operation.name)
```
%% Cell type:code id: tags:
``` python
print("--- Service Processes ---")
print("there are "+str(len(wps.processes))+" processes available for execution")
for process in wps.processes:
print(process.identifier, process.title, process.abstract)
```
%% Cell type:code id: tags:
``` python
def wps_execute(user_id, algo_wps_xml):
print("--- EXECUTE---")
url = copa_backend_url + 'wps?userId=' + user_id
tree = ET.parse(algo_wps_xml)
response = requests.post(url, headers = \
{'Authorization': 'Bearer '+oauth_token, 'Content-Type': 'application/xml'}, \
data=ET.tostring(tree.getroot()))
print(response.text)
tree = ET.fromstring(response.text)
return tree.find('{http://www.opengis.net/wps/2.0}JobID').text
dirname = os.getcwd()
read_xml = os.path.join(dirname, 'wps_compact.xml')
print("--- EXECUTE---")
i = 0
time1 = time.time()
job = []
jobIds = []
while i < 6:
jobId = wps_execute(user_id, read_xml)
job.append(jobId)
jobIds.append(jobId)
time.sleep(1)
i += 1
while len(job) != 0:
for Id in job:
url = copa_backend_url + 'wps/getStatus?userId=' + user_id
body = { "jobID": Id }
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
parsed = json.loads(response.text)
if parsed["status"] == "SUCCEEDED":
job.remove(Id)
time.sleep(1)
time2 = time.time()
final_time = (time2 - time1)
print('execution time was', final_time)
```
%% Cell type:code id: tags:
``` python
print("--- GetStatus---")
for Id in jobIds:
#FIXME retrieve the id of the user
url = copa_backend_url + 'wps/getStatus?userId=' + user_id
body = { "jobID": Id }
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
parsed = json.loads(response.text)
print(json.dumps(parsed, indent=2, sort_keys=True))
```
%% Cell type:code id: tags:
``` python
print("--- GetResult---")
for Id in jobIds:
#FIXME retrieve the id of the user
url = copa_backend_url + 'wps/'+ Id +'?userId=' + user_id
response = requests.get(url, headers = {'Authorization': 'Bearer '+oauth_token})
print(response.text)
```
%% Cell type:code id: tags:
``` python
print("--- Delete all jobs---")
for Id in jobIds:
url = copa_backend_url + 'wps/dismiss?userId=' + user_id
body = { "jobID": Id }
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
parsed = json.loads(response.text)
print(json.dumps(parsed, indent=2, sort_keys=True))
```
%% Cell type:code id: tags:
``` python
```
......
wps_notebook/Resample.ipynb
+ 1
− 1
View file @ 07825da2
%% Cell type:markdown id: tags:
## Prerequisite:
Run the following command in a terminal before launching the notebook:
`conda install -y --freeze-installed --file requirements.txt`
NB : make sure to be in the folder where you have the requirements.txt file
%% Cell type:code id: tags:
``` python
import requests
import json
import sys
import os
import xml.etree.ElementTree as ET
import time
from wpsToolsBox import wpsTB
import glob
import numpy as np
import configparser
try:
import wget
except :
!pip install wget
import wget
try:
import rasterio
except:
!pip install rasterio
import rasterio
```
%% Cell type:code id: tags:
``` python
#Load the file that contains auth values
config = configparser.ConfigParser()
config.read('/projects/.maap/auth.ini')
#Retrieve auth values
user_id = config['auth']['user_id']
email = config['auth']['email']
password = config['auth']['password']
#Set env variables
os.environ["EMAIL"] = email
os.environ["PASSWORD"] = password
```
%% Cell type:code id: tags:
``` python
#Load the file that contains maap values
config = configparser.ConfigParser()
config.read('/projects/.maap/maap.ini')
#Retrieve maap values
copa_backend_url = config['maap']['copa_backend_url']
CLIENT_ID = config['maap']['client_id']
url_token = config['maap']['url_token']
url_gravitee_s3 = config['maap']['url_gravitee_s3']
```
%% Cell type:code id: tags:
``` python
response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "openid+profile"})
response = requests.post(url_token, data=data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "profile"})
data = json.loads(response.text)
print(data)
oauth_token = data['access_token']
```
%% Cell type:code id: tags:
``` python
wps = wpsTB(oauth_token, copa_backend_url, user_id)
```
%% Cell type:code id: tags:
``` python
wps.wps_processes()
```
%% Cell type:code id: tags:
``` python
wps.wps_process_info('resample')
```
%% Cell type:code id: tags:
``` python
dir_path = '/projects/data/resample/'
if not os.path.exists(dir_path):
os.makedirs(dir_path)
```
%% Cell type:code id: tags:
``` python
!maap-s3.py login $EMAIL $PASSWORD
```
%% Cell type:code id: tags:
``` python
!maap-s3.py list 'maap-scientific-data/shared/demo/ratio'
```
%% Cell type:code id: tags:
``` python
while len(glob.glob('/projects/data/resample/ratio.tiff')) == 0:
!maap-s3.py download 'maap-scientific-data/shared/demo/ratio.tiff' '/projects/data/resample/ratio.tiff'
time.sleep(5)
ratio_path = '/projects/data/resample/ratio.tiff'
dir_path = '/projects/data/resample/'
path_s3 = 'maap-scientific-data/shared/demo/ratio.tiff'
```
%% Cell type:code id: tags:
``` python
url = url_gravitee_s3+path_s3
response = requests.get(url, headers = {'Authorization': 'Bearer '+oauth_token}, allow_redirects=False)
time.sleep(5)
print(response.text)
url_ratio = response.headers['Location']
print('url ratio :', url_ratio)
```
%% Cell type:code id: tags:
``` python
dirname = os.getcwd()
xml_file = os.path.join(dirname, 'xml_file/wps_resample.xml')
xml_file_out = os.path.join(dirname, 'resample_ratio.xml')
wps.add_value(xml_file, 'size', '2', xml_file_out)
wps.add_value(xml_file_out, 'input_image', url_ratio, xml_file_out)
wps.add_value(xml_file_out, 'output', dir_path, xml_file_out)
```
%% Cell type:code id: tags:
``` python
jobID = wps.wps_execute_process(xml_file_out, 1)
```
%% Cell type:code id: tags:
``` python
print("--- GetStatus---")
#FIXME retrieve the id of the user
url = copa_backend_url + 'wps/getStatus?userId=' + user_id
body = { "jobID": jobID[0] }
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
parsed = json.loads(response.text)
print(json.dumps(parsed, indent=2, sort_keys=True))
```
%% Cell type:code id: tags:
``` python
print("--- GetResult---")
#FIXME retrieve the id of the user
url = copa_backend_url + 'wps/'+ jobID[0] +'?userId=' + user_id
response = requests.get(url, headers = {'Authorization': 'Bearer '+oauth_token})
print(response.text)
```
%% Cell type:code id: tags:
``` python
response = wps.wps_result(jobID[0])
url_ratio_resample = wps.get_url(response, 'ratio_resample.npy')
print(url_ratio_resample)
print("Url ratio_resample.npy :", url_ratio_resample)
ratio_resample_path = wget.download(url_ratio_resample, '/projects/data/resample/ratio_resample.npy')
```
%% Cell type:code id: tags:
``` python
ratio_resample_arr = np.load(ratio_resample_path, allow_pickle=True)
ratio_arr = rasterio.open(ratio_path).read(1)
print("ratio shape :", ratio_arr.shape)
print("ratio resample shape :", ratio_resample_arr.shape)
```
%% Cell type:code id: tags:
``` python
import matplotlib.pyplot as plt
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13,13))
ax1.imshow(ratio_arr, vmin=0, vmax=0.8)
ax1.set_title('ratio')
ax2.imshow(ratio_resample_arr, vmin=0, vmax=0.8)
ax2.set_title('ratio resample (x2)')
```
%% Cell type:code id: tags:
``` python
print("---DeleteJob---")
url = copa_backend_url + 'wps/dismiss?userId=' + user_id
body = { "jobID": jobID[0]}
#body = { "jobID": '62054cac310b5900011f25f0'}
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
print(response.text)
```
%% Cell type:code id: tags:
``` python
```
......
wps_notebook/Scalability_Demonstration.ipynb
+ 1
− 1
View file @ 07825da2
%% Cell type:code id: tags:
``` python
import requests
import json
import sys
import os
import xml.etree.ElementTree as ET
import time
import sys
from wpsToolsBox import wpsTB
import configparser
import pandas as pd
```
%% Cell type:code id: tags:
``` python
#Load the file that contains auth values
config = configparser.ConfigParser()
config.read('/projects/.maap/auth.ini')
#Retrieve auth values
user_id = config['auth']['user_id']
email = config['auth']['email']
password = config['auth']['password']
```
%% Cell type:code id: tags:
``` python
#Load the file that contains maap values
config = configparser.ConfigParser()
config.read('/projects/.maap/maap.ini')
#Retrieve maap values
copa_backend_url = config['maap']['copa_backend_url']
CLIENT_ID = config['maap']['client_id']
url_token = config['maap']['url_token']
```
%% Cell type:code id: tags:
``` python
response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "openid+profile"})
response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "profile"})
data = json.loads(response.text)
print(data)
oauth_token = data['access_token']
```
%% Cell type:code id: tags:
``` python
wps = wpsTB(oauth_token, copa_backend_url, user_id)
```
%% Cell type:code id: tags:
``` python
wps.wps_info()
```
%% Cell type:code id: tags:
``` python
wps.wps_processes()
```
%% Cell type:code id: tags:
``` python
wps.wps_process_info('readstwo')
```
%% Cell type:code id: tags:
``` python
dirname = os.getcwd()
xml_file = os.path.join(dirname, 'xml_file/wps_readstwo.xml')
jobID = wps.wps_execute_process(xml_file, 1)
```
%% Cell type:code id: tags:
``` python
print("--- GetStatus---")
for Id in jobID:
#FIXME retrieve the id of the user
url = copa_backend_url + 'wps/getStatus?userId=' + user_id
body = { "jobID": Id }
print(Id)
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
parsed = json.loads(response.text)
print(json.dumps(parsed, indent=2, sort_keys=True))
```
%% Cell type:code id: tags:
``` python
print("--- GetResult---")
for Id in jobID:
#FIXME retrieve the id of the user
url = copa_backend_url + 'wps/'+ Id +'?userId=' + user_id
response = requests.get(url, headers = {'Authorization': 'Bearer '+oauth_token})
print(response.text)
```
%% Cell type:code id: tags:
``` python
print("--- Delete all jobs---")
df2 = pd.read_csv("jobIDs.csv")
print('List of the backuped job IDs')
jobs=[]
for i in df2.values :
jobs.append(i[0])
print(jobs)
#jobID =['62221dcb310b59000170059d','62221c61310b590001700599','62221c8d310b59000170059b']
for Id in jobID:
url = copa_backend_url + 'wps/dismiss?userId=' + user_id
body = { "jobID": Id }
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
parsed = json.loads(response.text)
print(json.dumps(parsed, indent=2, sort_keys=True))
```
%% Cell type:code id: tags:
``` python
```
......
wps_notebook/Scalability_aksk_biopal_fh.ipynb
+ 1
− 1
View file @ 07825da2
%% Cell type:code id: tags:
``` python
import requests
import json
import sys
import os
import xml.etree.ElementTree as ET
import time
import sys
from wpsToolsBox import wpsTB
import configparser
import pandas as pd
```
%% Cell type:code id: tags:
``` python
#Load the file that contains auth values
config = configparser.ConfigParser()
config.read('/projects/.maap/auth.ini')
#Retrieve auth values
user_id = config['auth']['user_id']
email = config['auth']['email']
password = config['auth']['password']
```
%% Cell type:code id: tags:
``` python
#Load the file that contains maap values
config = configparser.ConfigParser()
config.read('/projects/.maap/maap.ini')
#Retrieve maap values
copa_backend_url = config['maap']['copa_backend_url']
CLIENT_ID = config['maap']['client_id']
url_token = config['maap']['url_token']
```
%% Cell type:code id: tags:
``` python
response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "openid+profile"})
response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "profile"})
data = json.loads(response.text)
print(data)
oauth_token = data['access_token']
```
%% Cell type:code id: tags:
``` python
wps = wpsTB(oauth_token, copa_backend_url, user_id)
```
%% Cell type:code id: tags:
``` python
wps.wps_info()
```
%% Cell type:code id: tags:
``` python
wps.wps_processes()
```
%% Cell type:code id: tags:
``` python
wps.wps_process_info('aksk-biopal-fh')
```
%% Cell type:code id: tags:
``` python
dirname = os.getcwd()
xml_file = os.path.join(dirname, 'xml_file/wps_aksk-biopal-fh.xml')
jobID = wps.wps_execute_process(xml_file, 100)
```
%% Cell type:code id: tags:
``` python
print("--- GetStatus---")
for Id in jobID:
#FIXME retrieve the id of the user
url = copa_backend_url + 'wps/getStatus?userId=' + user_id
body = { "jobID": Id }
print(Id)
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
parsed = json.loads(response.text)
print(json.dumps(parsed, indent=2, sort_keys=True))
```
%% Cell type:code id: tags:
``` python
print("--- GetResult---")
for Id in jobID:
#FIXME retrieve the id of the user
url = copa_backend_url + 'wps/'+ Id +'?userId=' + user_id
response = requests.get(url, headers = {'Authorization': 'Bearer '+oauth_token})
print(response.text)
```
%% Cell type:code id: tags:
``` python
print("--- Delete all jobs---")
df2 = pd.read_csv("jobIDs.csv")
print('List of the backuped job IDs')
jobs=[]
for i in df2.values :
jobs.append(i[0])
print(jobs)
#jobID =['623c840d4f4c630001aca946', '623c840f4f4c630001aca948', '623c84104f4c630001aca94a', '623c84124f4c630001aca94c', '623c84144f4c630001aca94e', '623c84154f4c630001aca950', '623c84174f4c630001aca952', '623c84184f4c630001aca954', '623c841a4f4c630001aca956', '623c841b4f4c630001aca958', '623c841d4f4c630001aca95a', '623c841e4f4c630001aca95c', '623c84204f4c630001aca95e', '623c84214f4c630001aca960', '623c84234f4c630001aca962', '623c84244f4c630001aca964', '623c84264f4c630001aca966', '623c84274f4c630001aca968', '623c84294f4c630001aca96a', '623c842a4f4c630001aca96c', '623c842c4f4c630001aca96e', '623c842d4f4c630001aca970', '623c842f4f4c630001aca972', '623c84304f4c630001aca974', '623c84324f4c630001aca976', '623c84334f4c630001aca978', '623c84354f4c630001aca97a', '623c84364f4c630001aca97c', '623c84384f4c630001aca97e', '623c84394f4c630001aca980', '623c843b4f4c630001aca982', '623c843c4f4c630001aca984', '623c843e4f4c630001aca986', '623c843f4f4c630001aca988', '623c84414f4c630001aca98a', '623c84424f4c630001aca98c', '623c84444f4c630001aca98e', '623c84454f4c630001aca990', '623c84474f4c630001aca992', '623c84484f4c630001aca994', '623c844a4f4c630001aca996', '623c844b4f4c630001aca998', '623c844d4f4c630001aca99a', '623c844e4f4c630001aca99c', '623c84504f4c630001aca99e', '623c84514f4c630001aca9a0', '623c84534f4c630001aca9a2', '623c84554f4c630001aca9a4', '623c84564f4c630001aca9a6', '623c84584f4c630001aca9a8', '623c84594f4c630001aca9aa', '623c845a4f4c630001aca9ac', '623c845c4f4c630001aca9ae', '623c845d4f4c630001aca9b0', '623c845f4f4c630001aca9b2', '623c84604f4c630001aca9b4', '623c84624f4c630001aca9b6', '623c84634f4c630001aca9b8', '623c84654f4c630001aca9ba', '623c84664f4c630001aca9bc', '623c84684f4c630001aca9be', '623c846a4f4c630001aca9c0', '623c846b4f4c630001aca9c2', '623c846d4f4c630001aca9c4', '623c846e4f4c630001aca9c6', '623c84704f4c630001aca9c8', '623c84714f4c630001aca9ca', '623c84734f4c630001aca9cc', '623c84744f4c630001aca9ce', '623c84764f4c630001aca9d0', '623c84774f4c630001aca9d2', '623c84794f4c630001aca9d4', '623c847a4f4c630001aca9d6', '623c847c4f4c630001aca9d8', '623c847d4f4c630001aca9da', '623c847f4f4c630001aca9dc', '623c84804f4c630001aca9de', '623c84824f4c630001aca9e0', '623c84834f4c630001aca9e2', '623c84854f4c630001aca9e4', '623c84864f4c630001aca9e6', '623c84884f4c630001aca9e8', '623c84894f4c630001aca9ea', '623c848b4f4c630001aca9ec', '623c848c4f4c630001aca9ee', '623c848e4f4c630001aca9f0', '623c848f4f4c630001aca9f2', '623c84914f4c630001aca9f4', '623c84924f4c630001aca9f6', '623c84944f4c630001aca9f8', '623c84954f4c630001aca9fa', '623c84974f4c630001aca9fc', '623c84984f4c630001aca9fe', '623c849a4f4c630001acaa00', '623c849b4f4c630001acaa02', '623c849d4f4c630001acaa04', '623c849e4f4c630001acaa06', '623c84a04f4c630001acaa08', '623c84a14f4c630001acaa0a', '623c84a34f4c630001acaa0c']
for Id in jobID:
url = copa_backend_url + 'wps/dismiss?userId=' + user_id
body = { "jobID": Id }
response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)
parsed = json.loads(response.text)
print(json.dumps(parsed, indent=2, sort_keys=True))
```
%% Cell type:code id: tags:
``` python
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment